Skip to content

ARROW-8147: [C++] add GCS library to ThirdpartyToolchain#11268

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
coryan wants to merge 4 commits into from
Closed

ARROW-8147: [C++] add GCS library to ThirdpartyToolchain #11268

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
a725c52
ARROW-8147: [C++] add GCS library to ThirdpartyToolchain
coryan Sep 29, 2021
da2e245
Address review comments
coryan Sep 30, 2021
4e1341e
Address review comments
coryan Oct 2, 2021
77d5184
Address review comments
coryan Oct 4, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions ci/docker/ubuntu-20.04-cpp.dockerfile
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -110,6 +110,7 @@ ENV ARROW_BUILD_TESTS=ON \
ARROW_DATASET=ON \
ARROW_FLIGHT=OFF \
ARROW_GANDIVA=ON \
ARROW_GCS=ON \
ARROW_HDFS=ON \
ARROW_HOME=/usr/local \
ARROW_INSTALL_NAME_RPATH=OFF \
Expand All@@ -129,6 +130,7 @@ ENV ARROW_BUILD_TESTS=ON \
ARROW_WITH_ZSTD=ON \
ASAN_SYMBOLIZER_PATH=/usr/lib/llvm-${llvm}/bin/llvm-symbolizer \
AWSSDK_SOURCE=BUNDLED \
google_cloud_cpp_storage_SOURCE=BUNDLED \

@pitroupitrouOct 5, 2021

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Was GCS_SOURCE too ambiguous? This is quite a long parameter name...

@koukouOct 5, 2021

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We use CMake's package name for XXX_SOURCE.
If we don't follow the convention, we can choose GCS_SOURCE.

@coryancoryanOct 5, 2021

Copy link
Copy Markdown
ContributorAuthor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Let me know how to proceed, I can do either.

@pitroupitrouOct 5, 2021

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If it's the CMake name then fair enough.

GTest_SOURCE=BUNDLED \
gRPC_SOURCE=BUNDLED \
ORC_SOURCE=BUNDLED \
Expand Down
2 changes: 2 additions & 0 deletions ci/scripts/cpp_build.sh
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -74,6 +74,7 @@ cmake -G "${CMAKE_GENERATOR:-Ninja}" \
-DARROW_GANDIVA_JAVA=${ARROW_GANDIVA_JAVA:-OFF} \
-DARROW_GANDIVA_PC_CXX_FLAGS=${ARROW_GANDIVA_PC_CXX_FLAGS:-} \
-DARROW_GANDIVA=${ARROW_GANDIVA:-OFF} \
-DARROW_GCS=${ARROW_GCS:-OFF} \
-DARROW_HDFS=${ARROW_HDFS:-ON} \
-DARROW_HIVESERVER2=${ARROW_HIVESERVER2:-OFF} \
-DARROW_INSTALL_NAME_RPATH=${ARROW_INSTALL_NAME_RPATH:-ON} \
Expand DownExpand Up@@ -121,6 +122,7 @@ cmake -G "${CMAKE_GENERATOR:-Ninja}" \
-DCMAKE_INSTALL_PREFIX=${CMAKE_INSTALL_PREFIX:-${ARROW_HOME}} \
-DCMAKE_UNITY_BUILD=${CMAKE_UNITY_BUILD:-OFF} \
-Dgflags_SOURCE=${gflags_SOURCE:-} \
-Dgoogle_cloud_cpp_storage_SOURCE=${google_cloud_cpp_storage_SOURCE:-} \
-DgRPC_SOURCE=${gRPC_SOURCE:-} \
-DGTest_SOURCE=${GTest_SOURCE:-} \
-DLz4_SOURCE=${Lz4_SOURCE:-} \
Expand Down
8 changes: 8 additions & 0 deletions cpp/CMakeLists.txt
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -740,6 +740,14 @@ if(ARROW_ORC)
endif()
endif()

if(ARROW_GCS)
list(APPEND ARROW_LINK_LIBS google-cloud-cpp::storage)
list(APPEND ARROW_STATIC_LINK_LIBS google-cloud-cpp::storage)
if(google_cloud_cpp_storage_SOURCE STREQUAL "SYSTEM")
list(APPEND ARROW_STATIC_INSTALL_INTERFACE_LIBS google-cloud-cpp::storage)
endif()
endif()

if(ARROW_USE_GLOG)
list(APPEND ARROW_LINK_LIBS glog::glog)
list(APPEND ARROW_STATIC_LINK_LIBS glog::glog)
Expand Down
4 changes: 4 additions & 0 deletions cpp/cmake_modules/DefineOptions.cmake
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -224,6 +224,10 @@ if("${CMAKE_SOURCE_DIR}" STREQUAL "${CMAKE_CURRENT_SOURCE_DIR}")

define_option(ARROW_GANDIVA "Build the Gandiva libraries" OFF)

define_option(ARROW_GCS
"Build Arrow with GCS support (requires the GCloud SDK for C++)" OFF)
mark_as_advanced(ARROW_GCS) # TODO(ARROW-1231) - remove once completed

define_option(ARROW_HDFS "Build the Arrow HDFS bridge" OFF)

define_option(ARROW_HIVESERVER2 "Build the HiveServer2 client and Arrow adapter" OFF)
Expand Down
221 changes: 221 additions & 0 deletions cpp/cmake_modules/ThirdpartyToolchain.cmake
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -151,6 +151,8 @@ macro(build_dependency DEPENDENCY_NAME)
build_gflags()
elseif("${DEPENDENCY_NAME}" STREQUAL "GLOG")
build_glog()
elseif("${DEPENDENCY_NAME}" STREQUAL "google_cloud_cpp_storage")
build_google_cloud_cpp_storage()
elseif("${DEPENDENCY_NAME}" STREQUAL "gRPC")
build_grpc()
elseif("${DEPENDENCY_NAME}" STREQUAL "GTest")
Expand DownExpand Up@@ -287,6 +289,10 @@ if(ARROW_FLIGHT)
set(ARROW_WITH_ZLIB ON)
endif()

if(ARROW_GCS)
set(ARROW_WITH_GOOGLE_CLOUD_CPP ON)
endif()

if(ARROW_JSON)
set(ARROW_WITH_RAPIDJSON ON)
endif()
Expand DownExpand Up@@ -433,6 +439,14 @@ else()
)
endif()

if(DEFINED ENV{ARROW_CRC32C_URL})
set(CRC32C_URL "$ENV{ARROW_CRC32C_URL}")
else()
set_urls(CRC32C_SOURCE_URL
"https://github.com/google/crc32c/archive/${ARROW_CRC32C_BUILD_VERSION}.tar.gz"
)
endif()

if(DEFINED ENV{ARROW_GBENCHMARK_URL})
set(GBENCHMARK_SOURCE_URL "$ENV{ARROW_GBENCHMARK_URL}")
else()
Expand DownExpand Up@@ -460,6 +474,14 @@ else()
)
endif()

if(DEFINED ENV{ARROW_GOOGLE_CLOUD_CPP_URL})
set(google_cloud_cpp_storage_SOURCE_URL "$ENV{ARROW_GOOGLE_CLOUD_CPP_URL}")
else()
set_urls(google_cloud_cpp_storage_SOURCE_URL
"https://github.com/googleapis/google-cloud-cpp/archive/${ARROW_GOOGLE_CLOUD_CPP_BUILD_VERSION}.tar.gz"
)
endif()

if(DEFINED ENV{ARROW_GRPC_URL})
set(GRPC_SOURCE_URL "$ENV{ARROW_GRPC_URL}")
else()
Expand DownExpand Up@@ -497,6 +519,14 @@ else()
)
endif()

if(DEFINED ENV{ARROW_NLOHMANN_JSON_URL})
set(NLOHMANN_JSON_SOURCE_URL "$ENV{ARROW_NLOHMANN_JSON_URL}")
else()
set_urls(NLOHMANN_JSON_SOURCE_URL
"https://github.com/nlohmann/json/archive/${ARROW_NLOHMANN_JSON_BUILD_VERSION}.tar.gz"
)
endif()

if(DEFINED ENV{ARROW_LZ4_URL})
set(LZ4_SOURCE_URL "$ENV{ARROW_LZ4_URL}")
else()
Expand DownExpand Up@@ -3470,6 +3500,197 @@ if(ARROW_WITH_GRPC)
endif()
endif()

# ----------------------------------------------------------------------
# GCS and dependencies

macro(build_crc32c_once)
if(NOT TARGET crc32c_ep)
message(STATUS "Building crc32c from source")
# Build crc32c
set(CRC32C_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/crc32c_ep-install")
set(CRC32C_CMAKE_ARGS
${EP_COMMON_CMAKE_ARGS}
-DCMAKE_INSTALL_LIBDIR=lib
"-DCMAKE_INSTALL_PREFIX=<INSTALL_DIR>"
-DCMAKE_CXX_STANDARD=11
-DCRC32C_BUILD_TESTS=OFF
-DCRC32C_BUILD_BENCHMARKS=OFF
-DCRC32C_USE_GLOG=OFF)

set(_CRC32C_STATIC_LIBRARY
"${CRC32C_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}crc32c${CMAKE_STATIC_LIBRARY_SUFFIX}"
)
set(CRC32C_BUILD_BYPRODUCTS ${_CRC32C_STATIC_LIBRARY})
set(CRC32C_LIBRARIES crc32c)

externalproject_add(crc32c_ep
${EP_LOG_OPTIONS}
INSTALL_DIR ${CRC32C_PREFIX}
URL ${CRC32C_SOURCE_URL}
URL_HASH "SHA256=${ARROW_CRC32C_BUILD_SHA256_CHECKSUM}"
CMAKE_ARGS ${CRC32C_CMAKE_ARGS}
BUILD_BYPRODUCTS ${CRC32C_BUILD_BYPRODUCTS})
add_library(Crc32c::crc32c STATIC IMPORTED)
set_target_properties(Crc32c::crc32c
PROPERTIES IMPORTED_LOCATION ${_CRC32C_STATIC_LIBRARY}
INTERFACE_INCLUDE_DIRECTORIES
"${CRC32C_INCLUDE}/include")
add_dependencies(Crc32c::crc32c crc32c_ep)
endif()
endmacro()

macro(build_nlohmann_json_once)
if(NOT TARGET nlohmann_json_ep)
message(STATUS "Building nlohmann-json from source")
# "Build" nlohmann-json
set(NLOHMANN_JSON_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/nlohmann_json_ep-install")
set(NLOHMANN_JSON_CMAKE_ARGS
${EP_COMMON_CMAKE_ARGS} -DCMAKE_CXX_STANDARD=11
"-DCMAKE_INSTALL_PREFIX=<INSTALL_DIR>" -DBUILD_TESTING=OFF)

set(NLOHMANN_JSON_BUILD_BYPRODUCTS ${NLOHMANN_JSON_PREFIX}/include/json.hpp)

externalproject_add(nlohmann_json_ep
${EP_LOG_OPTIONS}
INSTALL_DIR ${NLOHMANN_JSON_PREFIX}
URL ${NLOHMANN_JSON_SOURCE_URL}
URL_HASH "SHA256=${ARROW_NLOHMANN_JSON_BUILD_SHA256_CHECKSUM}"
CMAKE_ARGS ${NLOHMANN_JSON_CMAKE_ARGS}
BUILD_BYPRODUCTS ${NLOHMANN_JSON_BUILD_BYPRODUCTS})
add_library(nlohmann_json::nlohmann_json INTERFACE IMPORTED)
set_target_properties(nlohmann_json::nlohmann_json
PROPERTIES INTERFACE_INCLUDE_DIRECTORIES
"${NLOHMANN_JSON_PREFIX}/include")
add_dependencies(nlohmann_json::nlohmann_json nlohmann_json_ep)
endif()
endmacro()

macro(build_google_cloud_cpp_storage)
message(STATUS "Building google-cloud-cpp from source")
message(STATUS "Only building the google-cloud-cpp::storage component")

# List of dependencies taken from https://github.com/googleapis/google-cloud-cpp/blob/master/doc/packaging.md
build_absl_once()
build_crc32c_once()
build_nlohmann_json_once()

# Curl is required on all platforms, but building it internally might also trip over S3's copy.
# For now, force its inclusion from the underlying system or fail.
find_package(CURL REQUIRED 7.47.0)

# Build google-cloud-cpp, with only storage_client

# Inject vendored packages via CMAKE_PREFIX_PATH
list(APPEND GOOGLE_CLOUD_CPP_PREFIX_PATH_LIST ${ABSL_PREFIX})
list(APPEND GOOGLE_CLOUD_CPP_PREFIX_PATH_LIST ${CRC32C_PREFIX})
list(APPEND GOOGLE_CLOUD_CPP_PREFIX_PATH_LIST ${NLOHMANN_JSON_PREFIX})

set(GOOGLE_CLOUD_CPP_PREFIX_PATH_LIST_SEP_CHAR "|")
list(JOIN GOOGLE_CLOUD_CPP_PREFIX_PATH_LIST
${GOOGLE_CLOUD_CPP_PREFIX_PATH_LIST_SEP_CHAR} GOOGLE_CLOUD_CPP_PREFIX_PATH)

set(GOOGLE_CLOUD_CPP_INSTALL_PREFIX
"${CMAKE_CURRENT_BINARY_DIR}/google_cloud_cpp_ep-install")
set(GOOGLE_CLOUD_CPP_INCLUDE_DIR "${GOOGLE_CLOUD_CPP_INSTALL_PREFIX}/include")
set(GOOGLE_CLOUD_CPP_CMAKE_ARGS
${EP_COMMON_CMAKE_ARGS}
-DBUILD_TESTING=OFF
-DCMAKE_INSTALL_LIBDIR=lib
"-DCMAKE_INSTALL_PREFIX=<INSTALL_DIR>"
-DCMAKE_INSTALL_RPATH=$ORIGIN
-DCMAKE_PREFIX_PATH=${GOOGLE_CLOUD_CPP_PREFIX_PATH}
# Compile only the storage library and its dependencies. To enable
# other services (Spanner, Bigtable, etc.) add them (as a list) to this
# parameter. Each has its own `google-cloud-cpp::*` library.
-DGOOGLE_CLOUD_CPP_ENABLE=storage)
if(OPENSSL_ROOT_DIR)
list(APPEND GOOGLE_CLOUD_CPP_CMAKE_ARGS -DOPENSSL_ROOT_DIR=${OPENSSL_ROOT_DIR})
endif()

add_custom_target(google_cloud_cpp_dependencies)

add_dependencies(google_cloud_cpp_dependencies absl_ep)
add_dependencies(google_cloud_cpp_dependencies crc32c_ep)
add_dependencies(google_cloud_cpp_dependencies nlohmann_json_ep)
# Typically the steps to build the AWKSSDK provide `CURL::libcurl`, but if that is
# disabled we need to provide our own.
if(NOT TARGET CURL::libcurl)
find_package(CURL REQUIRED)
if(NOT TARGET CURL::libcurl)
# For CMake 3.11 or older
add_library(CURL::libcurl UNKNOWN IMPORTED)
set_target_properties(CURL::libcurl
PROPERTIES INTERFACE_INCLUDE_DIRECTORIES
"${CURL_INCLUDE_DIRS}" IMPORTED_LOCATION
"${CURL_LIBRARIES}")
endif()
endif()

set(GOOGLE_CLOUD_CPP_STATIC_LIBRARY_STORAGE
"${GOOGLE_CLOUD_CPP_INSTALL_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}google_cloud_cpp_storage${CMAKE_STATIC_LIBRARY_SUFFIX}"
)

set(GOOGLE_CLOUD_CPP_STATIC_LIBRARY_COMMON
"${GOOGLE_CLOUD_CPP_INSTALL_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}google_cloud_cpp_common${CMAKE_STATIC_LIBRARY_SUFFIX}"
)

externalproject_add(google_cloud_cpp_ep
${EP_LOG_OPTIONS}
LIST_SEPARATOR ${GOOGLE_CLOUD_CPP_PREFIX_PATH_LIST_SEP_CHAR}
INSTALL_DIR ${GOOGLE_CLOUD_CPP_INSTALL_PREFIX}
URL ${google_cloud_cpp_storage_SOURCE_URL}
URL_HASH "SHA256=${ARROW_GOOGLE_CLOUD_CPP_BUILD_SHA256_CHECKSUM}"
CMAKE_ARGS ${GOOGLE_CLOUD_CPP_CMAKE_ARGS}
BUILD_BYPRODUCTS ${GOOGLE_CLOUD_CPP_STATIC_LIBRARY_STORAGE}
${GOOGLE_CLOUD_CPP_STATIC_LIBRARY_COMMON}
DEPENDS google_cloud_cpp_dependencies)
add_dependencies(toolchain google_cloud_cpp_ep)

add_library(google-cloud-cpp::common STATIC IMPORTED)
set_target_properties(google-cloud-cpp::common
PROPERTIES IMPORTED_LOCATION
"${GOOGLE_CLOUD_CPP_STATIC_LIBRARY_COMMON}"
INTERFACE_INCLUDE_DIRECTORIES
"${GOOGLE_CLOUD_CPP_INCLUDE_DIR}")
set_property(TARGET google-cloud-cpp::common
PROPERTY INTERFACE_LINK_LIBRARIES
absl::any
absl::flat_hash_map
absl::memory
absl::optional
absl::time
Threads::Threads)

add_library(google-cloud-cpp::storage STATIC IMPORTED)
set_target_properties(google-cloud-cpp::storage
PROPERTIES IMPORTED_LOCATION
"${GOOGLE_CLOUD_CPP_STATIC_LIBRARY_STORAGE}"
INTERFACE_INCLUDE_DIRECTORIES
"${GOOGLE_CLOUD_CPP_INCLUDE_DIR}")
set_property(TARGET google-cloud-cpp::storage
PROPERTY INTERFACE_LINK_LIBRARIES
google-cloud-cpp::common
absl::memory
absl::strings
absl::str_format
absl::time
absl::variant
nlohmann_json::nlohmann_json
Crc32c::crc32c
CURL::libcurl
Threads::Threads
OpenSSL::SSL
OpenSSL::Crypto)
add_dependencies(google-cloud-cpp::storage google_cloud_cpp_ep)

list(APPEND ARROW_BUNDLED_STATIC_LIBS google-cloud-cpp::storage
google-cloud-cpp::common)
endmacro()

if(ARROW_WITH_GOOGLE_CLOUD_CPP)
resolve_dependency(google_cloud_cpp_storage)
endif()

#
# HDFS thirdparty setup

Expand Down
9 changes: 9 additions & 0 deletions cpp/thirdparty/versions.txt
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -41,12 +41,16 @@ ARROW_BZIP2_BUILD_VERSION=1.0.8
ARROW_BZIP2_BUILD_SHA256_CHECKSUM=ab5a03176ee106d3f0fa90e381da478ddae405918153cca248e682cd0c4a2269
ARROW_CARES_BUILD_VERSION=1.17.1
ARROW_CARES_BUILD_SHA256_CHECKSUM=d73dd0f6de824afd407ce10750ea081af47eba52b8a6cb307d220131ad93fc40
ARROW_CRC32C_BUILD_VERSION=1.1.1
ARROW_CRC32C_BUILD_SHA256_CHECKSUM=a6533f45b1670b5d59b38a514d82b09c6fb70cc1050467220216335e873074e8
ARROW_GBENCHMARK_BUILD_VERSION=v1.5.2
ARROW_GBENCHMARK_BUILD_SHA256_CHECKSUM=dccbdab796baa1043f04982147e67bb6e118fe610da2c65f88912d73987e700c
ARROW_GFLAGS_BUILD_VERSION=v2.2.2
ARROW_GFLAGS_BUILD_SHA256_CHECKSUM=34af2f15cf7367513b352bdcd2493ab14ce43692d2dcd9dfc499492966c64dcf
ARROW_GLOG_BUILD_VERSION=v0.4.0
ARROW_GLOG_BUILD_SHA256_CHECKSUM=f28359aeba12f30d73d9e4711ef356dc842886968112162bc73002645139c39c
ARROW_GOOGLE_CLOUD_CPP_BUILD_VERSION=v1.31.1
ARROW_GOOGLE_CLOUD_CPP_BUILD_SHA256_CHECKSUM=dc7cbf95b506a84b48cf71e0462985d262183edeaabdacaaee2109852394a609
ARROW_GRPC_BUILD_VERSION=v1.35.0
ARROW_GRPC_BUILD_SHA256_CHECKSUM=27dd2fc5c9809ddcde8eb6fa1fa278a3486566dfc28335fca13eb8df8bd3b958
ARROW_GTEST_BUILD_VERSION=1.11.0
Expand All@@ -59,6 +63,8 @@ ARROW_LZ4_BUILD_SHA256_CHECKSUM=030644df4611007ff7dc962d981f390361e6c97a34e5cbc3
# https://github.com/microsoft/mimalloc/issues/353
ARROW_MIMALLOC_BUILD_VERSION=v1.7.2
ARROW_MIMALLOC_BUILD_SHA256_CHECKSUM=b1912e354565a4b698410f7583c0f83934a6dbb3ade54ab7ddcb1569320936bd
ARROW_NLOHMANN_JSON_BUILD_VERSION=v3.10.2
ARROW_NLOHMANN_JSON_BUILD_SHA256_CHECKSUM=081ed0f9f89805c2d96335c3acfa993b39a0a5b4b4cef7edb68dd2210a13458c
ARROW_ORC_BUILD_VERSION=1.7.0
ARROW_ORC_BUILD_SHA256_CHECKSUM=45d6ba9149ffa2aaa168d61ab326f61181861c94529f26da3918a9aa2f801e39
ARROW_PROTOBUF_BUILD_VERSION=v3.17.3
Expand DownExpand Up@@ -98,14 +104,17 @@ DEPENDENCIES=(
"ARROW_BROTLI_URL brotli-${ARROW_BROTLI_BUILD_VERSION}.tar.gz https://github.com/google/brotli/archive/${ARROW_BROTLI_BUILD_VERSION}.tar.gz"
"ARROW_BZIP2_URL bzip2-${ARROW_BZIP2_BUILD_VERSION}.tar.gz https://sourceware.org/pub/bzip2/bzip2-${ARROW_BZIP2_BUILD_VERSION}.tar.gz"
"ARROW_CARES_URL cares-${ARROW_CARES_BUILD_VERSION}.tar.gz https://c-ares.haxx.se/download/c-ares-${ARROW_CARES_BUILD_VERSION}.tar.gz"
"ARROW_CRC32C_URL crc32c-${ARROW_CRC32C_BUILD_VERSION}.tar.gz https://github.com/google/crc32c/archive/refs/tags/${ARROW_CRC32C_BUILD_VERSION}.tar.gz
"ARROW_GBENCHMARK_URL gbenchmark-${ARROW_GBENCHMARK_BUILD_VERSION}.tar.gz https://github.com/google/benchmark/archive/${ARROW_GBENCHMARK_BUILD_VERSION}.tar.gz"
"ARROW_GFLAGS_URL gflags-${ARROW_GFLAGS_BUILD_VERSION}.tar.gz https://github.com/gflags/gflags/archive/${ARROW_GFLAGS_BUILD_VERSION}.tar.gz"
"ARROW_GLOG_URL glog-${ARROW_GLOG_BUILD_VERSION}.tar.gz https://github.com/google/glog/archive/${ARROW_GLOG_BUILD_VERSION}.tar.gz"
"ARROW_GOOGLE_CLOUD_CPP_URL google-cloud-cpp-${ARROW_GOOGLE_CLOUD_CPP_BUILD_VERSION}.tar.gz https://github.com/googleapis/google-cloud-cpp/archive/refs/tags/${ARROW_GOOGLE_CLOUD_CPP_BUILD_VERSION}.tar.gz
"ARROW_GRPC_URL grpc-${ARROW_GRPC_BUILD_VERSION}.tar.gz https://github.com/grpc/grpc/archive/${ARROW_GRPC_BUILD_VERSION}.tar.gz"
"ARROW_GTEST_URL gtest-${ARROW_GTEST_BUILD_VERSION}.tar.gz https://github.com/google/googletest/archive/release-${ARROW_GTEST_BUILD_VERSION}.tar.gz"
"ARROW_JEMALLOC_URL jemalloc-${ARROW_JEMALLOC_BUILD_VERSION}.tar.bz2 https://github.com/jemalloc/jemalloc/releases/download/${ARROW_JEMALLOC_BUILD_VERSION}/jemalloc-${ARROW_JEMALLOC_BUILD_VERSION}.tar.bz2"
"ARROW_LZ4_URL lz4-${ARROW_LZ4_BUILD_VERSION}.tar.gz https://github.com/lz4/lz4/archive/${ARROW_LZ4_BUILD_VERSION}.tar.gz"
"ARROW_MIMALLOC_URL mimalloc-${ARROW_MIMALLOC_BUILD_VERSION}.tar.gz https://github.com/microsoft/mimalloc/archive/${ARROW_MIMALLOC_BUILD_VERSION}.tar.gz"
"ARROW_NLOHMANN_JSON_URL nlohmann-json-${ARROW_NLOHMANN_JSON_BUILD_VERSION}.tar.gz https://github.com/nlohmann/json/archive/refs/tags/${ARROW_NLOHMANN_JSON_BUILD_VERSION}.tar.gz
"ARROW_ORC_URL orc-${ARROW_ORC_BUILD_VERSION}.tar.gz https://github.com/apache/orc/archive/rel/release-${ARROW_ORC_BUILD_VERSION}.tar.gz"
"ARROW_PROTOBUF_URL protobuf-${ARROW_PROTOBUF_BUILD_VERSION}.tar.gz https://github.com/google/protobuf/releases/download/${ARROW_PROTOBUF_BUILD_VERSION}/protobuf-all-${ARROW_PROTOBUF_BUILD_VERSION:1}.tar.gz"
"ARROW_RAPIDJSON_URL rapidjson-${ARROW_RAPIDJSON_BUILD_VERSION}.tar.gz https://github.com/miloyip/rapidjson/archive/${ARROW_RAPIDJSON_BUILD_VERSION}.tar.gz"
Expand Down
, 'i'); if (__m === '*' || __re.test(location.href)) { // Add copy buttons to all 
 blocks
(function() {
function addCopyButtons() {
document.querySelectorAll('pre code').forEach(function(codeBlock) {
if (codeBlock.parentElement.hasAttribute('data-copy-added')) return;
codeBlock.parentElement.setAttribute('data-copy-added', 'true');
var btn = document.createElement('button');
btn.textContent = 'Copy';
btn.style.cssText = 'position:absolute;top:4px;right:4px;padding:2px 8px;font-size:11px;background:#4ecdc4;border:none;border-radius:4px;color:#1a1a2e;cursor:pointer;opacity:0.7;transition:opacity 0.2s;';
btn.onmouseover = function() { this.style.opacity = '1'; };
btn.onmouseout = function() { this.style.opacity = '0.7'; };
btn.onclick = function() {
navigator.clipboard.writeText(codeBlock.textContent).then(function() {
btn.textContent = 'Copied!';
setTimeout(function() { btn.textContent = 'Copy'; }, 1500);
});
};
codeBlock.parentElement.style.position = 'relative';
codeBlock.parentElement.appendChild(btn);
});
}
addCopyButtons();
// Re-run on dynamic content
var observer = new MutationObserver(addCopyButtons);
observer.observe(document.body, { childList: true, subtree: true });
})();
}
} catch(__e) { console.warn('[Userscript:Add Copy Buttons to Code Blocks]', __e); }
})();
(function(){
try {
var __m = "github.com";
var __re = new RegExp('^' + "github\\.com" + '
ARROW-8147: [C++] add GCS library to ThirdpartyToolchain by coryan · Pull Request #11268 · apache/arrow · GitHub
Skip to content
ARROW-8147: [C++] add GCS library to ThirdpartyToolchain#11268

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Closed

coryan
wants to merge
4
commits into

from

 Closed


ARROW-8147: [C++] add GCS library to ThirdpartyToolchain
#11268
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


4 commits


Select commit
Hold shift + click to select a range
a725c52

ARROW-8147: [C++] add GCS library to ThirdpartyToolchain


coryan Sep 29, 2021
da2e245

Address review comments


coryan Sep 30, 2021
4e1341e

Address review comments


coryan Oct 2, 2021
77d5184

Address review comments


coryan Oct 4, 2021

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


2 changes: 2 additions & 0 deletions
ci/docker/ubuntu-20.04-cpp.dockerfile

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -110,6 +110,7 @@ ENV ARROW_BUILD_TESTS=ON \
 ARROW_DATASET=ON \
 ARROW_FLIGHT=OFF \
 ARROW_GANDIVA=ON \
 ARROW_GCS=ON \
 ARROW_HDFS=ON \
 ARROW_HOME=/usr/local \
 ARROW_INSTALL_NAME_RPATH=OFF \
Expand All@@ -129,6 +130,7 @@ ENV ARROW_BUILD_TESTS=ON \
 ARROW_WITH_ZSTD=ON \
 ASAN_SYMBOLIZER_PATH=/usr/lib/llvm-${llvm}/bin/llvm-symbolizer \
 AWSSDK_SOURCE=BUNDLED \
 google_cloud_cpp_storage_SOURCE=BUNDLED \
@pitroupitrouOct 5, 2021

Copy link


Copy Markdown

Member
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Was GCS_SOURCE too ambiguous? This is quite a long parameter name...
@koukouOct 5, 2021

Copy link


Copy Markdown

Member
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We use CMake's package name for XXX_SOURCE.

If we don't follow the convention, we can choose GCS_SOURCE.
@coryancoryanOct 5, 2021

Copy link


Copy Markdown

ContributorAuthor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Let me know how to proceed, I can do either.
@pitroupitrouOct 5, 2021

Copy link


Copy Markdown

Member
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If it's the CMake name then fair enough.
 GTest_SOURCE=BUNDLED \
 gRPC_SOURCE=BUNDLED \
 ORC_SOURCE=BUNDLED \
Expand Down

2 changes: 2 additions & 0 deletions
ci/scripts/cpp_build.sh

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -74,6 +74,7 @@ cmake -G "${CMAKE_GENERATOR:-Ninja}" \
 -DARROW_GANDIVA_JAVA=${ARROW_GANDIVA_JAVA:-OFF} \
 -DARROW_GANDIVA_PC_CXX_FLAGS=${ARROW_GANDIVA_PC_CXX_FLAGS:-} \
 -DARROW_GANDIVA=${ARROW_GANDIVA:-OFF} \
 -DARROW_GCS=${ARROW_GCS:-OFF} \
 -DARROW_HDFS=${ARROW_HDFS:-ON} \
 -DARROW_HIVESERVER2=${ARROW_HIVESERVER2:-OFF} \
 -DARROW_INSTALL_NAME_RPATH=${ARROW_INSTALL_NAME_RPATH:-ON} \
Expand DownExpand Up@@ -121,6 +122,7 @@ cmake -G "${CMAKE_GENERATOR:-Ninja}" \
 -DCMAKE_INSTALL_PREFIX=${CMAKE_INSTALL_PREFIX:-${ARROW_HOME}} \
 -DCMAKE_UNITY_BUILD=${CMAKE_UNITY_BUILD:-OFF} \
 -Dgflags_SOURCE=${gflags_SOURCE:-} \
 -Dgoogle_cloud_cpp_storage_SOURCE=${google_cloud_cpp_storage_SOURCE:-} \
 -DgRPC_SOURCE=${gRPC_SOURCE:-} \
 -DGTest_SOURCE=${GTest_SOURCE:-} \
 -DLz4_SOURCE=${Lz4_SOURCE:-} \
Expand Down

8 changes: 8 additions & 0 deletions
cpp/CMakeLists.txt

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -740,6 +740,14 @@ if(ARROW_ORC)
 endif()
endif()

if(ARROW_GCS)
 list(APPEND ARROW_LINK_LIBS google-cloud-cpp::storage)
 list(APPEND ARROW_STATIC_LINK_LIBS google-cloud-cpp::storage)
 if(google_cloud_cpp_storage_SOURCE STREQUAL "SYSTEM")
 list(APPEND ARROW_STATIC_INSTALL_INTERFACE_LIBS google-cloud-cpp::storage)
 endif()
endif()

if(ARROW_USE_GLOG)
 list(APPEND ARROW_LINK_LIBS glog::glog)
 list(APPEND ARROW_STATIC_LINK_LIBS glog::glog)
Expand Down

4 changes: 4 additions & 0 deletions
cpp/cmake_modules/DefineOptions.cmake

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -224,6 +224,10 @@ if("${CMAKE_SOURCE_DIR}" STREQUAL "${CMAKE_CURRENT_SOURCE_DIR}")

 define_option(ARROW_GANDIVA "Build the Gandiva libraries" OFF)

 define_option(ARROW_GCS
 "Build Arrow with GCS support (requires the GCloud SDK for C++)" OFF)
 mark_as_advanced(ARROW_GCS) # TODO(ARROW-1231) - remove once completed

 define_option(ARROW_HDFS "Build the Arrow HDFS bridge" OFF)

 define_option(ARROW_HIVESERVER2 "Build the HiveServer2 client and Arrow adapter" OFF)
Expand Down

221 changes: 221 additions & 0 deletions
cpp/cmake_modules/ThirdpartyToolchain.cmake

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -151,6 +151,8 @@ macro(build_dependency DEPENDENCY_NAME)
 build_gflags()
 elseif("${DEPENDENCY_NAME}" STREQUAL "GLOG")
 build_glog()
 elseif("${DEPENDENCY_NAME}" STREQUAL "google_cloud_cpp_storage")
 build_google_cloud_cpp_storage()
 elseif("${DEPENDENCY_NAME}" STREQUAL "gRPC")
 build_grpc()
 elseif("${DEPENDENCY_NAME}" STREQUAL "GTest")
Expand DownExpand Up@@ -287,6 +289,10 @@ if(ARROW_FLIGHT)
 set(ARROW_WITH_ZLIB ON)
endif()

if(ARROW_GCS)
 set(ARROW_WITH_GOOGLE_CLOUD_CPP ON)
endif()

if(ARROW_JSON)
 set(ARROW_WITH_RAPIDJSON ON)
endif()
Expand DownExpand Up@@ -433,6 +439,14 @@ else()
 )
endif()

if(DEFINED ENV{ARROW_CRC32C_URL})
 set(CRC32C_URL "$ENV{ARROW_CRC32C_URL}")
else()
 set_urls(CRC32C_SOURCE_URL
 "https://github.com/google/crc32c/archive/${ARROW_CRC32C_BUILD_VERSION}.tar.gz"
 )
endif()

if(DEFINED ENV{ARROW_GBENCHMARK_URL})
 set(GBENCHMARK_SOURCE_URL "$ENV{ARROW_GBENCHMARK_URL}")
else()
Expand DownExpand Up@@ -460,6 +474,14 @@ else()
 )
endif()

if(DEFINED ENV{ARROW_GOOGLE_CLOUD_CPP_URL})
 set(google_cloud_cpp_storage_SOURCE_URL "$ENV{ARROW_GOOGLE_CLOUD_CPP_URL}")
else()
 set_urls(google_cloud_cpp_storage_SOURCE_URL
 "https://github.com/googleapis/google-cloud-cpp/archive/${ARROW_GOOGLE_CLOUD_CPP_BUILD_VERSION}.tar.gz"
 )
endif()

if(DEFINED ENV{ARROW_GRPC_URL})
 set(GRPC_SOURCE_URL "$ENV{ARROW_GRPC_URL}")
else()
Expand DownExpand Up@@ -497,6 +519,14 @@ else()
 )
endif()

if(DEFINED ENV{ARROW_NLOHMANN_JSON_URL})
 set(NLOHMANN_JSON_SOURCE_URL "$ENV{ARROW_NLOHMANN_JSON_URL}")
else()
 set_urls(NLOHMANN_JSON_SOURCE_URL
 "https://github.com/nlohmann/json/archive/${ARROW_NLOHMANN_JSON_BUILD_VERSION}.tar.gz"
 )
endif()

if(DEFINED ENV{ARROW_LZ4_URL})
 set(LZ4_SOURCE_URL "$ENV{ARROW_LZ4_URL}")
else()
Expand DownExpand Up@@ -3470,6 +3500,197 @@ if(ARROW_WITH_GRPC)
 endif()
endif()

# ----------------------------------------------------------------------
# GCS and dependencies

macro(build_crc32c_once)
 if(NOT TARGET crc32c_ep)
 message(STATUS "Building crc32c from source")
 # Build crc32c
 set(CRC32C_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/crc32c_ep-install")
 set(CRC32C_CMAKE_ARGS
 ${EP_COMMON_CMAKE_ARGS}
 -DCMAKE_INSTALL_LIBDIR=lib
 "-DCMAKE_INSTALL_PREFIX=<INSTALL_DIR>"
 -DCMAKE_CXX_STANDARD=11
 -DCRC32C_BUILD_TESTS=OFF
 -DCRC32C_BUILD_BENCHMARKS=OFF
 -DCRC32C_USE_GLOG=OFF)

 set(_CRC32C_STATIC_LIBRARY
 "${CRC32C_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}crc32c${CMAKE_STATIC_LIBRARY_SUFFIX}"
 )
 set(CRC32C_BUILD_BYPRODUCTS ${_CRC32C_STATIC_LIBRARY})
 set(CRC32C_LIBRARIES crc32c)

 externalproject_add(crc32c_ep
 ${EP_LOG_OPTIONS}
 INSTALL_DIR ${CRC32C_PREFIX}
 URL ${CRC32C_SOURCE_URL}
 URL_HASH "SHA256=${ARROW_CRC32C_BUILD_SHA256_CHECKSUM}"
 CMAKE_ARGS ${CRC32C_CMAKE_ARGS}
 BUILD_BYPRODUCTS ${CRC32C_BUILD_BYPRODUCTS})
 add_library(Crc32c::crc32c STATIC IMPORTED)
 set_target_properties(Crc32c::crc32c
 PROPERTIES IMPORTED_LOCATION ${_CRC32C_STATIC_LIBRARY}
 INTERFACE_INCLUDE_DIRECTORIES
 "${CRC32C_INCLUDE}/include")
 add_dependencies(Crc32c::crc32c crc32c_ep)
 endif()
endmacro()

macro(build_nlohmann_json_once)
 if(NOT TARGET nlohmann_json_ep)
 message(STATUS "Building nlohmann-json from source")
 # "Build" nlohmann-json
 set(NLOHMANN_JSON_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/nlohmann_json_ep-install")
 set(NLOHMANN_JSON_CMAKE_ARGS
 ${EP_COMMON_CMAKE_ARGS} -DCMAKE_CXX_STANDARD=11
 "-DCMAKE_INSTALL_PREFIX=<INSTALL_DIR>" -DBUILD_TESTING=OFF)

 set(NLOHMANN_JSON_BUILD_BYPRODUCTS ${NLOHMANN_JSON_PREFIX}/include/json.hpp)

 externalproject_add(nlohmann_json_ep
 ${EP_LOG_OPTIONS}
 INSTALL_DIR ${NLOHMANN_JSON_PREFIX}
 URL ${NLOHMANN_JSON_SOURCE_URL}
 URL_HASH "SHA256=${ARROW_NLOHMANN_JSON_BUILD_SHA256_CHECKSUM}"
 CMAKE_ARGS ${NLOHMANN_JSON_CMAKE_ARGS}
 BUILD_BYPRODUCTS ${NLOHMANN_JSON_BUILD_BYPRODUCTS})
 add_library(nlohmann_json::nlohmann_json INTERFACE IMPORTED)
 set_target_properties(nlohmann_json::nlohmann_json
 PROPERTIES INTERFACE_INCLUDE_DIRECTORIES
 "${NLOHMANN_JSON_PREFIX}/include")
 add_dependencies(nlohmann_json::nlohmann_json nlohmann_json_ep)
 endif()
endmacro()

macro(build_google_cloud_cpp_storage)
 message(STATUS "Building google-cloud-cpp from source")
 message(STATUS "Only building the google-cloud-cpp::storage component")

 # List of dependencies taken from https://github.com/googleapis/google-cloud-cpp/blob/master/doc/packaging.md
 build_absl_once()
 build_crc32c_once()
 build_nlohmann_json_once()

 # Curl is required on all platforms, but building it internally might also trip over S3's copy.
 # For now, force its inclusion from the underlying system or fail.
 find_package(CURL REQUIRED 7.47.0)

 # Build google-cloud-cpp, with only storage_client

 # Inject vendored packages via CMAKE_PREFIX_PATH
 list(APPEND GOOGLE_CLOUD_CPP_PREFIX_PATH_LIST ${ABSL_PREFIX})
 list(APPEND GOOGLE_CLOUD_CPP_PREFIX_PATH_LIST ${CRC32C_PREFIX})
 list(APPEND GOOGLE_CLOUD_CPP_PREFIX_PATH_LIST ${NLOHMANN_JSON_PREFIX})

 set(GOOGLE_CLOUD_CPP_PREFIX_PATH_LIST_SEP_CHAR "|")
 list(JOIN GOOGLE_CLOUD_CPP_PREFIX_PATH_LIST
 ${GOOGLE_CLOUD_CPP_PREFIX_PATH_LIST_SEP_CHAR} GOOGLE_CLOUD_CPP_PREFIX_PATH)

 set(GOOGLE_CLOUD_CPP_INSTALL_PREFIX
 "${CMAKE_CURRENT_BINARY_DIR}/google_cloud_cpp_ep-install")
 set(GOOGLE_CLOUD_CPP_INCLUDE_DIR "${GOOGLE_CLOUD_CPP_INSTALL_PREFIX}/include")
 set(GOOGLE_CLOUD_CPP_CMAKE_ARGS
 ${EP_COMMON_CMAKE_ARGS}
 -DBUILD_TESTING=OFF
 -DCMAKE_INSTALL_LIBDIR=lib
 "-DCMAKE_INSTALL_PREFIX=<INSTALL_DIR>"
 -DCMAKE_INSTALL_RPATH=$ORIGIN
 -DCMAKE_PREFIX_PATH=${GOOGLE_CLOUD_CPP_PREFIX_PATH}
 # Compile only the storage library and its dependencies. To enable
 # other services (Spanner, Bigtable, etc.) add them (as a list) to this
 # parameter. Each has its own `google-cloud-cpp::*` library.
 -DGOOGLE_CLOUD_CPP_ENABLE=storage)
 if(OPENSSL_ROOT_DIR)
 list(APPEND GOOGLE_CLOUD_CPP_CMAKE_ARGS -DOPENSSL_ROOT_DIR=${OPENSSL_ROOT_DIR})
 endif()

 add_custom_target(google_cloud_cpp_dependencies)

 add_dependencies(google_cloud_cpp_dependencies absl_ep)
 add_dependencies(google_cloud_cpp_dependencies crc32c_ep)
 add_dependencies(google_cloud_cpp_dependencies nlohmann_json_ep)
 # Typically the steps to build the AWKSSDK provide `CURL::libcurl`, but if that is
 # disabled we need to provide our own.
 if(NOT TARGET CURL::libcurl)
 find_package(CURL REQUIRED)
 if(NOT TARGET CURL::libcurl)
 # For CMake 3.11 or older
 add_library(CURL::libcurl UNKNOWN IMPORTED)
 set_target_properties(CURL::libcurl
 PROPERTIES INTERFACE_INCLUDE_DIRECTORIES
 "${CURL_INCLUDE_DIRS}" IMPORTED_LOCATION
 "${CURL_LIBRARIES}")
 endif()
 endif()

 set(GOOGLE_CLOUD_CPP_STATIC_LIBRARY_STORAGE
 "${GOOGLE_CLOUD_CPP_INSTALL_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}google_cloud_cpp_storage${CMAKE_STATIC_LIBRARY_SUFFIX}"
 )

 set(GOOGLE_CLOUD_CPP_STATIC_LIBRARY_COMMON
 "${GOOGLE_CLOUD_CPP_INSTALL_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}google_cloud_cpp_common${CMAKE_STATIC_LIBRARY_SUFFIX}"
 )

 externalproject_add(google_cloud_cpp_ep
 ${EP_LOG_OPTIONS}
 LIST_SEPARATOR ${GOOGLE_CLOUD_CPP_PREFIX_PATH_LIST_SEP_CHAR}
 INSTALL_DIR ${GOOGLE_CLOUD_CPP_INSTALL_PREFIX}
 URL ${google_cloud_cpp_storage_SOURCE_URL}
 URL_HASH "SHA256=${ARROW_GOOGLE_CLOUD_CPP_BUILD_SHA256_CHECKSUM}"
 CMAKE_ARGS ${GOOGLE_CLOUD_CPP_CMAKE_ARGS}
 BUILD_BYPRODUCTS ${GOOGLE_CLOUD_CPP_STATIC_LIBRARY_STORAGE}
 ${GOOGLE_CLOUD_CPP_STATIC_LIBRARY_COMMON}
 DEPENDS google_cloud_cpp_dependencies)
 add_dependencies(toolchain google_cloud_cpp_ep)

 add_library(google-cloud-cpp::common STATIC IMPORTED)
 set_target_properties(google-cloud-cpp::common
 PROPERTIES IMPORTED_LOCATION
 "${GOOGLE_CLOUD_CPP_STATIC_LIBRARY_COMMON}"
 INTERFACE_INCLUDE_DIRECTORIES
 "${GOOGLE_CLOUD_CPP_INCLUDE_DIR}")
 set_property(TARGET google-cloud-cpp::common
 PROPERTY INTERFACE_LINK_LIBRARIES
 absl::any
 absl::flat_hash_map
 absl::memory
 absl::optional
 absl::time
 Threads::Threads)

 add_library(google-cloud-cpp::storage STATIC IMPORTED)
 set_target_properties(google-cloud-cpp::storage
 PROPERTIES IMPORTED_LOCATION
 "${GOOGLE_CLOUD_CPP_STATIC_LIBRARY_STORAGE}"
 INTERFACE_INCLUDE_DIRECTORIES
 "${GOOGLE_CLOUD_CPP_INCLUDE_DIR}")
 set_property(TARGET google-cloud-cpp::storage
 PROPERTY INTERFACE_LINK_LIBRARIES
 google-cloud-cpp::common
 absl::memory
 absl::strings
 absl::str_format
 absl::time
 absl::variant
 nlohmann_json::nlohmann_json
 Crc32c::crc32c
 CURL::libcurl
 Threads::Threads
 OpenSSL::SSL
 OpenSSL::Crypto)
 add_dependencies(google-cloud-cpp::storage google_cloud_cpp_ep)

 list(APPEND ARROW_BUNDLED_STATIC_LIBS google-cloud-cpp::storage
 google-cloud-cpp::common)
endmacro()

if(ARROW_WITH_GOOGLE_CLOUD_CPP)
 resolve_dependency(google_cloud_cpp_storage)
endif()

#
# HDFS thirdparty setup

Expand Down

9 changes: 9 additions & 0 deletions
cpp/thirdparty/versions.txt

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -41,12 +41,16 @@ ARROW_BZIP2_BUILD_VERSION=1.0.8
ARROW_BZIP2_BUILD_SHA256_CHECKSUM=ab5a03176ee106d3f0fa90e381da478ddae405918153cca248e682cd0c4a2269
ARROW_CARES_BUILD_VERSION=1.17.1
ARROW_CARES_BUILD_SHA256_CHECKSUM=d73dd0f6de824afd407ce10750ea081af47eba52b8a6cb307d220131ad93fc40
ARROW_CRC32C_BUILD_VERSION=1.1.1
ARROW_CRC32C_BUILD_SHA256_CHECKSUM=a6533f45b1670b5d59b38a514d82b09c6fb70cc1050467220216335e873074e8
ARROW_GBENCHMARK_BUILD_VERSION=v1.5.2
ARROW_GBENCHMARK_BUILD_SHA256_CHECKSUM=dccbdab796baa1043f04982147e67bb6e118fe610da2c65f88912d73987e700c
ARROW_GFLAGS_BUILD_VERSION=v2.2.2
ARROW_GFLAGS_BUILD_SHA256_CHECKSUM=34af2f15cf7367513b352bdcd2493ab14ce43692d2dcd9dfc499492966c64dcf
ARROW_GLOG_BUILD_VERSION=v0.4.0
ARROW_GLOG_BUILD_SHA256_CHECKSUM=f28359aeba12f30d73d9e4711ef356dc842886968112162bc73002645139c39c
ARROW_GOOGLE_CLOUD_CPP_BUILD_VERSION=v1.31.1
ARROW_GOOGLE_CLOUD_CPP_BUILD_SHA256_CHECKSUM=dc7cbf95b506a84b48cf71e0462985d262183edeaabdacaaee2109852394a609
ARROW_GRPC_BUILD_VERSION=v1.35.0
ARROW_GRPC_BUILD_SHA256_CHECKSUM=27dd2fc5c9809ddcde8eb6fa1fa278a3486566dfc28335fca13eb8df8bd3b958
ARROW_GTEST_BUILD_VERSION=1.11.0
Expand All@@ -59,6 +63,8 @@ ARROW_LZ4_BUILD_SHA256_CHECKSUM=030644df4611007ff7dc962d981f390361e6c97a34e5cbc3
# https://github.com/microsoft/mimalloc/issues/353
ARROW_MIMALLOC_BUILD_VERSION=v1.7.2
ARROW_MIMALLOC_BUILD_SHA256_CHECKSUM=b1912e354565a4b698410f7583c0f83934a6dbb3ade54ab7ddcb1569320936bd
ARROW_NLOHMANN_JSON_BUILD_VERSION=v3.10.2
ARROW_NLOHMANN_JSON_BUILD_SHA256_CHECKSUM=081ed0f9f89805c2d96335c3acfa993b39a0a5b4b4cef7edb68dd2210a13458c
ARROW_ORC_BUILD_VERSION=1.7.0
ARROW_ORC_BUILD_SHA256_CHECKSUM=45d6ba9149ffa2aaa168d61ab326f61181861c94529f26da3918a9aa2f801e39
ARROW_PROTOBUF_BUILD_VERSION=v3.17.3
Expand DownExpand Up@@ -98,14 +104,17 @@ DEPENDENCIES=(
 "ARROW_BROTLI_URL brotli-${ARROW_BROTLI_BUILD_VERSION}.tar.gz https://github.com/google/brotli/archive/${ARROW_BROTLI_BUILD_VERSION}.tar.gz"
 "ARROW_BZIP2_URL bzip2-${ARROW_BZIP2_BUILD_VERSION}.tar.gz https://sourceware.org/pub/bzip2/bzip2-${ARROW_BZIP2_BUILD_VERSION}.tar.gz"
 "ARROW_CARES_URL cares-${ARROW_CARES_BUILD_VERSION}.tar.gz https://c-ares.haxx.se/download/c-ares-${ARROW_CARES_BUILD_VERSION}.tar.gz"
 "ARROW_CRC32C_URL crc32c-${ARROW_CRC32C_BUILD_VERSION}.tar.gz https://github.com/google/crc32c/archive/refs/tags/${ARROW_CRC32C_BUILD_VERSION}.tar.gz
 "ARROW_GBENCHMARK_URL gbenchmark-${ARROW_GBENCHMARK_BUILD_VERSION}.tar.gz https://github.com/google/benchmark/archive/${ARROW_GBENCHMARK_BUILD_VERSION}.tar.gz"
 "ARROW_GFLAGS_URL gflags-${ARROW_GFLAGS_BUILD_VERSION}.tar.gz https://github.com/gflags/gflags/archive/${ARROW_GFLAGS_BUILD_VERSION}.tar.gz"
 "ARROW_GLOG_URL glog-${ARROW_GLOG_BUILD_VERSION}.tar.gz https://github.com/google/glog/archive/${ARROW_GLOG_BUILD_VERSION}.tar.gz"
 "ARROW_GOOGLE_CLOUD_CPP_URL google-cloud-cpp-${ARROW_GOOGLE_CLOUD_CPP_BUILD_VERSION}.tar.gz https://github.com/googleapis/google-cloud-cpp/archive/refs/tags/${ARROW_GOOGLE_CLOUD_CPP_BUILD_VERSION}.tar.gz
 "ARROW_GRPC_URL grpc-${ARROW_GRPC_BUILD_VERSION}.tar.gz https://github.com/grpc/grpc/archive/${ARROW_GRPC_BUILD_VERSION}.tar.gz"
 "ARROW_GTEST_URL gtest-${ARROW_GTEST_BUILD_VERSION}.tar.gz https://github.com/google/googletest/archive/release-${ARROW_GTEST_BUILD_VERSION}.tar.gz"
 "ARROW_JEMALLOC_URL jemalloc-${ARROW_JEMALLOC_BUILD_VERSION}.tar.bz2 https://github.com/jemalloc/jemalloc/releases/download/${ARROW_JEMALLOC_BUILD_VERSION}/jemalloc-${ARROW_JEMALLOC_BUILD_VERSION}.tar.bz2"
 "ARROW_LZ4_URL lz4-${ARROW_LZ4_BUILD_VERSION}.tar.gz https://github.com/lz4/lz4/archive/${ARROW_LZ4_BUILD_VERSION}.tar.gz"
 "ARROW_MIMALLOC_URL mimalloc-${ARROW_MIMALLOC_BUILD_VERSION}.tar.gz https://github.com/microsoft/mimalloc/archive/${ARROW_MIMALLOC_BUILD_VERSION}.tar.gz"
 "ARROW_NLOHMANN_JSON_URL nlohmann-json-${ARROW_NLOHMANN_JSON_BUILD_VERSION}.tar.gz https://github.com/nlohmann/json/archive/refs/tags/${ARROW_NLOHMANN_JSON_BUILD_VERSION}.tar.gz
 "ARROW_ORC_URL orc-${ARROW_ORC_BUILD_VERSION}.tar.gz https://github.com/apache/orc/archive/rel/release-${ARROW_ORC_BUILD_VERSION}.tar.gz"
 "ARROW_PROTOBUF_URL protobuf-${ARROW_PROTOBUF_BUILD_VERSION}.tar.gz https://github.com/google/protobuf/releases/download/${ARROW_PROTOBUF_BUILD_VERSION}/protobuf-all-${ARROW_PROTOBUF_BUILD_VERSION:1}.tar.gz"
 "ARROW_RAPIDJSON_URL rapidjson-${ARROW_RAPIDJSON_BUILD_VERSION}.tar.gz https://github.com/miloyip/rapidjson/archive/${ARROW_RAPIDJSON_BUILD_VERSION}.tar.gz"
Expand Down

, 'i');
if (__m === '*' || __re.test(location.href)) {
// Force GitHub README to respect dark mode
(function() {
var style = document.createElement('style');
style.textContent = '
.markdown-body {
color-scheme: dark light;
}
.markdown-body pre { background: #161b22 !important; }
.markdown-body code { background: rgba(110, 118, 129, 0.4) !important; }
.markdown-body table th, .markdown-body table td { border-color: #30363d !important; }
.markdown-body img { background: #0d1117; }
.markdown-body blockquote { border-left-color: #8b949e; }
.markdown-body hr { border-color: #30363d; }
';
document.head.appendChild(style);
})();
}
} catch(__e) { console.warn('[Userscript:GitHub Dark Mode README Fix]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '
ARROW-8147: [C++] add GCS library to ThirdpartyToolchain by coryan · Pull Request #11268 · apache/arrow · GitHub
Skip to content
ARROW-8147: [C++] add GCS library to ThirdpartyToolchain#11268

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Closed

coryan
wants to merge
4
commits into

from

 Closed


ARROW-8147: [C++] add GCS library to ThirdpartyToolchain
#11268
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


4 commits


Select commit
Hold shift + click to select a range
a725c52

ARROW-8147: [C++] add GCS library to ThirdpartyToolchain


coryan Sep 29, 2021
da2e245

Address review comments


coryan Sep 30, 2021
4e1341e

Address review comments


coryan Oct 2, 2021
77d5184

Address review comments


coryan Oct 4, 2021

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


2 changes: 2 additions & 0 deletions
ci/docker/ubuntu-20.04-cpp.dockerfile

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -110,6 +110,7 @@ ENV ARROW_BUILD_TESTS=ON \
 ARROW_DATASET=ON \
 ARROW_FLIGHT=OFF \
 ARROW_GANDIVA=ON \
 ARROW_GCS=ON \
 ARROW_HDFS=ON \
 ARROW_HOME=/usr/local \
 ARROW_INSTALL_NAME_RPATH=OFF \
Expand All@@ -129,6 +130,7 @@ ENV ARROW_BUILD_TESTS=ON \
 ARROW_WITH_ZSTD=ON \
 ASAN_SYMBOLIZER_PATH=/usr/lib/llvm-${llvm}/bin/llvm-symbolizer \
 AWSSDK_SOURCE=BUNDLED \
 google_cloud_cpp_storage_SOURCE=BUNDLED \
@pitroupitrouOct 5, 2021

Copy link


Copy Markdown

Member
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Was GCS_SOURCE too ambiguous? This is quite a long parameter name...
@koukouOct 5, 2021

Copy link


Copy Markdown

Member
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We use CMake's package name for XXX_SOURCE.

If we don't follow the convention, we can choose GCS_SOURCE.
@coryancoryanOct 5, 2021

Copy link


Copy Markdown

ContributorAuthor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Let me know how to proceed, I can do either.
@pitroupitrouOct 5, 2021

Copy link


Copy Markdown

Member
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If it's the CMake name then fair enough.
 GTest_SOURCE=BUNDLED \
 gRPC_SOURCE=BUNDLED \
 ORC_SOURCE=BUNDLED \
Expand Down

2 changes: 2 additions & 0 deletions
ci/scripts/cpp_build.sh

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -74,6 +74,7 @@ cmake -G "${CMAKE_GENERATOR:-Ninja}" \
 -DARROW_GANDIVA_JAVA=${ARROW_GANDIVA_JAVA:-OFF} \
 -DARROW_GANDIVA_PC_CXX_FLAGS=${ARROW_GANDIVA_PC_CXX_FLAGS:-} \
 -DARROW_GANDIVA=${ARROW_GANDIVA:-OFF} \
 -DARROW_GCS=${ARROW_GCS:-OFF} \
 -DARROW_HDFS=${ARROW_HDFS:-ON} \
 -DARROW_HIVESERVER2=${ARROW_HIVESERVER2:-OFF} \
 -DARROW_INSTALL_NAME_RPATH=${ARROW_INSTALL_NAME_RPATH:-ON} \
Expand DownExpand Up@@ -121,6 +122,7 @@ cmake -G "${CMAKE_GENERATOR:-Ninja}" \
 -DCMAKE_INSTALL_PREFIX=${CMAKE_INSTALL_PREFIX:-${ARROW_HOME}} \
 -DCMAKE_UNITY_BUILD=${CMAKE_UNITY_BUILD:-OFF} \
 -Dgflags_SOURCE=${gflags_SOURCE:-} \
 -Dgoogle_cloud_cpp_storage_SOURCE=${google_cloud_cpp_storage_SOURCE:-} \
 -DgRPC_SOURCE=${gRPC_SOURCE:-} \
 -DGTest_SOURCE=${GTest_SOURCE:-} \
 -DLz4_SOURCE=${Lz4_SOURCE:-} \
Expand Down

8 changes: 8 additions & 0 deletions
cpp/CMakeLists.txt

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -740,6 +740,14 @@ if(ARROW_ORC)
 endif()
endif()

if(ARROW_GCS)
 list(APPEND ARROW_LINK_LIBS google-cloud-cpp::storage)
 list(APPEND ARROW_STATIC_LINK_LIBS google-cloud-cpp::storage)
 if(google_cloud_cpp_storage_SOURCE STREQUAL "SYSTEM")
 list(APPEND ARROW_STATIC_INSTALL_INTERFACE_LIBS google-cloud-cpp::storage)
 endif()
endif()

if(ARROW_USE_GLOG)
 list(APPEND ARROW_LINK_LIBS glog::glog)
 list(APPEND ARROW_STATIC_LINK_LIBS glog::glog)
Expand Down

4 changes: 4 additions & 0 deletions
cpp/cmake_modules/DefineOptions.cmake

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -224,6 +224,10 @@ if("${CMAKE_SOURCE_DIR}" STREQUAL "${CMAKE_CURRENT_SOURCE_DIR}")

 define_option(ARROW_GANDIVA "Build the Gandiva libraries" OFF)

 define_option(ARROW_GCS
 "Build Arrow with GCS support (requires the GCloud SDK for C++)" OFF)
 mark_as_advanced(ARROW_GCS) # TODO(ARROW-1231) - remove once completed

 define_option(ARROW_HDFS "Build the Arrow HDFS bridge" OFF)

 define_option(ARROW_HIVESERVER2 "Build the HiveServer2 client and Arrow adapter" OFF)
Expand Down

221 changes: 221 additions & 0 deletions
cpp/cmake_modules/ThirdpartyToolchain.cmake

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -151,6 +151,8 @@ macro(build_dependency DEPENDENCY_NAME)
 build_gflags()
 elseif("${DEPENDENCY_NAME}" STREQUAL "GLOG")
 build_glog()
 elseif("${DEPENDENCY_NAME}" STREQUAL "google_cloud_cpp_storage")
 build_google_cloud_cpp_storage()
 elseif("${DEPENDENCY_NAME}" STREQUAL "gRPC")
 build_grpc()
 elseif("${DEPENDENCY_NAME}" STREQUAL "GTest")
Expand DownExpand Up@@ -287,6 +289,10 @@ if(ARROW_FLIGHT)
 set(ARROW_WITH_ZLIB ON)
endif()

if(ARROW_GCS)
 set(ARROW_WITH_GOOGLE_CLOUD_CPP ON)
endif()

if(ARROW_JSON)
 set(ARROW_WITH_RAPIDJSON ON)
endif()
Expand DownExpand Up@@ -433,6 +439,14 @@ else()
 )
endif()

if(DEFINED ENV{ARROW_CRC32C_URL})
 set(CRC32C_URL "$ENV{ARROW_CRC32C_URL}")
else()
 set_urls(CRC32C_SOURCE_URL
 "https://github.com/google/crc32c/archive/${ARROW_CRC32C_BUILD_VERSION}.tar.gz"
 )
endif()

if(DEFINED ENV{ARROW_GBENCHMARK_URL})
 set(GBENCHMARK_SOURCE_URL "$ENV{ARROW_GBENCHMARK_URL}")
else()
Expand DownExpand Up@@ -460,6 +474,14 @@ else()
 )
endif()

if(DEFINED ENV{ARROW_GOOGLE_CLOUD_CPP_URL})
 set(google_cloud_cpp_storage_SOURCE_URL "$ENV{ARROW_GOOGLE_CLOUD_CPP_URL}")
else()
 set_urls(google_cloud_cpp_storage_SOURCE_URL
 "https://github.com/googleapis/google-cloud-cpp/archive/${ARROW_GOOGLE_CLOUD_CPP_BUILD_VERSION}.tar.gz"
 )
endif()

if(DEFINED ENV{ARROW_GRPC_URL})
 set(GRPC_SOURCE_URL "$ENV{ARROW_GRPC_URL}")
else()
Expand DownExpand Up@@ -497,6 +519,14 @@ else()
 )
endif()

if(DEFINED ENV{ARROW_NLOHMANN_JSON_URL})
 set(NLOHMANN_JSON_SOURCE_URL "$ENV{ARROW_NLOHMANN_JSON_URL}")
else()
 set_urls(NLOHMANN_JSON_SOURCE_URL
 "https://github.com/nlohmann/json/archive/${ARROW_NLOHMANN_JSON_BUILD_VERSION}.tar.gz"
 )
endif()

if(DEFINED ENV{ARROW_LZ4_URL})
 set(LZ4_SOURCE_URL "$ENV{ARROW_LZ4_URL}")
else()
Expand DownExpand Up@@ -3470,6 +3500,197 @@ if(ARROW_WITH_GRPC)
 endif()
endif()

# ----------------------------------------------------------------------
# GCS and dependencies

macro(build_crc32c_once)
 if(NOT TARGET crc32c_ep)
 message(STATUS "Building crc32c from source")
 # Build crc32c
 set(CRC32C_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/crc32c_ep-install")
 set(CRC32C_CMAKE_ARGS
 ${EP_COMMON_CMAKE_ARGS}
 -DCMAKE_INSTALL_LIBDIR=lib
 "-DCMAKE_INSTALL_PREFIX=<INSTALL_DIR>"
 -DCMAKE_CXX_STANDARD=11
 -DCRC32C_BUILD_TESTS=OFF
 -DCRC32C_BUILD_BENCHMARKS=OFF
 -DCRC32C_USE_GLOG=OFF)

 set(_CRC32C_STATIC_LIBRARY
 "${CRC32C_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}crc32c${CMAKE_STATIC_LIBRARY_SUFFIX}"
 )
 set(CRC32C_BUILD_BYPRODUCTS ${_CRC32C_STATIC_LIBRARY})
 set(CRC32C_LIBRARIES crc32c)

 externalproject_add(crc32c_ep
 ${EP_LOG_OPTIONS}
 INSTALL_DIR ${CRC32C_PREFIX}
 URL ${CRC32C_SOURCE_URL}
 URL_HASH "SHA256=${ARROW_CRC32C_BUILD_SHA256_CHECKSUM}"
 CMAKE_ARGS ${CRC32C_CMAKE_ARGS}
 BUILD_BYPRODUCTS ${CRC32C_BUILD_BYPRODUCTS})
 add_library(Crc32c::crc32c STATIC IMPORTED)
 set_target_properties(Crc32c::crc32c
 PROPERTIES IMPORTED_LOCATION ${_CRC32C_STATIC_LIBRARY}
 INTERFACE_INCLUDE_DIRECTORIES
 "${CRC32C_INCLUDE}/include")
 add_dependencies(Crc32c::crc32c crc32c_ep)
 endif()
endmacro()

macro(build_nlohmann_json_once)
 if(NOT TARGET nlohmann_json_ep)
 message(STATUS "Building nlohmann-json from source")
 # "Build" nlohmann-json
 set(NLOHMANN_JSON_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/nlohmann_json_ep-install")
 set(NLOHMANN_JSON_CMAKE_ARGS
 ${EP_COMMON_CMAKE_ARGS} -DCMAKE_CXX_STANDARD=11
 "-DCMAKE_INSTALL_PREFIX=<INSTALL_DIR>" -DBUILD_TESTING=OFF)

 set(NLOHMANN_JSON_BUILD_BYPRODUCTS ${NLOHMANN_JSON_PREFIX}/include/json.hpp)

 externalproject_add(nlohmann_json_ep
 ${EP_LOG_OPTIONS}
 INSTALL_DIR ${NLOHMANN_JSON_PREFIX}
 URL ${NLOHMANN_JSON_SOURCE_URL}
 URL_HASH "SHA256=${ARROW_NLOHMANN_JSON_BUILD_SHA256_CHECKSUM}"
 CMAKE_ARGS ${NLOHMANN_JSON_CMAKE_ARGS}
 BUILD_BYPRODUCTS ${NLOHMANN_JSON_BUILD_BYPRODUCTS})
 add_library(nlohmann_json::nlohmann_json INTERFACE IMPORTED)
 set_target_properties(nlohmann_json::nlohmann_json
 PROPERTIES INTERFACE_INCLUDE_DIRECTORIES
 "${NLOHMANN_JSON_PREFIX}/include")
 add_dependencies(nlohmann_json::nlohmann_json nlohmann_json_ep)
 endif()
endmacro()

macro(build_google_cloud_cpp_storage)
 message(STATUS "Building google-cloud-cpp from source")
 message(STATUS "Only building the google-cloud-cpp::storage component")

 # List of dependencies taken from https://github.com/googleapis/google-cloud-cpp/blob/master/doc/packaging.md
 build_absl_once()
 build_crc32c_once()
 build_nlohmann_json_once()

 # Curl is required on all platforms, but building it internally might also trip over S3's copy.
 # For now, force its inclusion from the underlying system or fail.
 find_package(CURL REQUIRED 7.47.0)

 # Build google-cloud-cpp, with only storage_client

 # Inject vendored packages via CMAKE_PREFIX_PATH
 list(APPEND GOOGLE_CLOUD_CPP_PREFIX_PATH_LIST ${ABSL_PREFIX})
 list(APPEND GOOGLE_CLOUD_CPP_PREFIX_PATH_LIST ${CRC32C_PREFIX})
 list(APPEND GOOGLE_CLOUD_CPP_PREFIX_PATH_LIST ${NLOHMANN_JSON_PREFIX})

 set(GOOGLE_CLOUD_CPP_PREFIX_PATH_LIST_SEP_CHAR "|")
 list(JOIN GOOGLE_CLOUD_CPP_PREFIX_PATH_LIST
 ${GOOGLE_CLOUD_CPP_PREFIX_PATH_LIST_SEP_CHAR} GOOGLE_CLOUD_CPP_PREFIX_PATH)

 set(GOOGLE_CLOUD_CPP_INSTALL_PREFIX
 "${CMAKE_CURRENT_BINARY_DIR}/google_cloud_cpp_ep-install")
 set(GOOGLE_CLOUD_CPP_INCLUDE_DIR "${GOOGLE_CLOUD_CPP_INSTALL_PREFIX}/include")
 set(GOOGLE_CLOUD_CPP_CMAKE_ARGS
 ${EP_COMMON_CMAKE_ARGS}
 -DBUILD_TESTING=OFF
 -DCMAKE_INSTALL_LIBDIR=lib
 "-DCMAKE_INSTALL_PREFIX=<INSTALL_DIR>"
 -DCMAKE_INSTALL_RPATH=$ORIGIN
 -DCMAKE_PREFIX_PATH=${GOOGLE_CLOUD_CPP_PREFIX_PATH}
 # Compile only the storage library and its dependencies. To enable
 # other services (Spanner, Bigtable, etc.) add them (as a list) to this
 # parameter. Each has its own `google-cloud-cpp::*` library.
 -DGOOGLE_CLOUD_CPP_ENABLE=storage)
 if(OPENSSL_ROOT_DIR)
 list(APPEND GOOGLE_CLOUD_CPP_CMAKE_ARGS -DOPENSSL_ROOT_DIR=${OPENSSL_ROOT_DIR})
 endif()

 add_custom_target(google_cloud_cpp_dependencies)

 add_dependencies(google_cloud_cpp_dependencies absl_ep)
 add_dependencies(google_cloud_cpp_dependencies crc32c_ep)
 add_dependencies(google_cloud_cpp_dependencies nlohmann_json_ep)
 # Typically the steps to build the AWKSSDK provide `CURL::libcurl`, but if that is
 # disabled we need to provide our own.
 if(NOT TARGET CURL::libcurl)
 find_package(CURL REQUIRED)
 if(NOT TARGET CURL::libcurl)
 # For CMake 3.11 or older
 add_library(CURL::libcurl UNKNOWN IMPORTED)
 set_target_properties(CURL::libcurl
 PROPERTIES INTERFACE_INCLUDE_DIRECTORIES
 "${CURL_INCLUDE_DIRS}" IMPORTED_LOCATION
 "${CURL_LIBRARIES}")
 endif()
 endif()

 set(GOOGLE_CLOUD_CPP_STATIC_LIBRARY_STORAGE
 "${GOOGLE_CLOUD_CPP_INSTALL_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}google_cloud_cpp_storage${CMAKE_STATIC_LIBRARY_SUFFIX}"
 )

 set(GOOGLE_CLOUD_CPP_STATIC_LIBRARY_COMMON
 "${GOOGLE_CLOUD_CPP_INSTALL_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}google_cloud_cpp_common${CMAKE_STATIC_LIBRARY_SUFFIX}"
 )

 externalproject_add(google_cloud_cpp_ep
 ${EP_LOG_OPTIONS}
 LIST_SEPARATOR ${GOOGLE_CLOUD_CPP_PREFIX_PATH_LIST_SEP_CHAR}
 INSTALL_DIR ${GOOGLE_CLOUD_CPP_INSTALL_PREFIX}
 URL ${google_cloud_cpp_storage_SOURCE_URL}
 URL_HASH "SHA256=${ARROW_GOOGLE_CLOUD_CPP_BUILD_SHA256_CHECKSUM}"
 CMAKE_ARGS ${GOOGLE_CLOUD_CPP_CMAKE_ARGS}
 BUILD_BYPRODUCTS ${GOOGLE_CLOUD_CPP_STATIC_LIBRARY_STORAGE}
 ${GOOGLE_CLOUD_CPP_STATIC_LIBRARY_COMMON}
 DEPENDS google_cloud_cpp_dependencies)
 add_dependencies(toolchain google_cloud_cpp_ep)

 add_library(google-cloud-cpp::common STATIC IMPORTED)
 set_target_properties(google-cloud-cpp::common
 PROPERTIES IMPORTED_LOCATION
 "${GOOGLE_CLOUD_CPP_STATIC_LIBRARY_COMMON}"
 INTERFACE_INCLUDE_DIRECTORIES
 "${GOOGLE_CLOUD_CPP_INCLUDE_DIR}")
 set_property(TARGET google-cloud-cpp::common
 PROPERTY INTERFACE_LINK_LIBRARIES
 absl::any
 absl::flat_hash_map
 absl::memory
 absl::optional
 absl::time
 Threads::Threads)

 add_library(google-cloud-cpp::storage STATIC IMPORTED)
 set_target_properties(google-cloud-cpp::storage
 PROPERTIES IMPORTED_LOCATION
 "${GOOGLE_CLOUD_CPP_STATIC_LIBRARY_STORAGE}"
 INTERFACE_INCLUDE_DIRECTORIES
 "${GOOGLE_CLOUD_CPP_INCLUDE_DIR}")
 set_property(TARGET google-cloud-cpp::storage
 PROPERTY INTERFACE_LINK_LIBRARIES
 google-cloud-cpp::common
 absl::memory
 absl::strings
 absl::str_format
 absl::time
 absl::variant
 nlohmann_json::nlohmann_json
 Crc32c::crc32c
 CURL::libcurl
 Threads::Threads
 OpenSSL::SSL
 OpenSSL::Crypto)
 add_dependencies(google-cloud-cpp::storage google_cloud_cpp_ep)

 list(APPEND ARROW_BUNDLED_STATIC_LIBS google-cloud-cpp::storage
 google-cloud-cpp::common)
endmacro()

if(ARROW_WITH_GOOGLE_CLOUD_CPP)
 resolve_dependency(google_cloud_cpp_storage)
endif()

#
# HDFS thirdparty setup

Expand Down

9 changes: 9 additions & 0 deletions
cpp/thirdparty/versions.txt

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -41,12 +41,16 @@ ARROW_BZIP2_BUILD_VERSION=1.0.8
ARROW_BZIP2_BUILD_SHA256_CHECKSUM=ab5a03176ee106d3f0fa90e381da478ddae405918153cca248e682cd0c4a2269
ARROW_CARES_BUILD_VERSION=1.17.1
ARROW_CARES_BUILD_SHA256_CHECKSUM=d73dd0f6de824afd407ce10750ea081af47eba52b8a6cb307d220131ad93fc40
ARROW_CRC32C_BUILD_VERSION=1.1.1
ARROW_CRC32C_BUILD_SHA256_CHECKSUM=a6533f45b1670b5d59b38a514d82b09c6fb70cc1050467220216335e873074e8
ARROW_GBENCHMARK_BUILD_VERSION=v1.5.2
ARROW_GBENCHMARK_BUILD_SHA256_CHECKSUM=dccbdab796baa1043f04982147e67bb6e118fe610da2c65f88912d73987e700c
ARROW_GFLAGS_BUILD_VERSION=v2.2.2
ARROW_GFLAGS_BUILD_SHA256_CHECKSUM=34af2f15cf7367513b352bdcd2493ab14ce43692d2dcd9dfc499492966c64dcf
ARROW_GLOG_BUILD_VERSION=v0.4.0
ARROW_GLOG_BUILD_SHA256_CHECKSUM=f28359aeba12f30d73d9e4711ef356dc842886968112162bc73002645139c39c
ARROW_GOOGLE_CLOUD_CPP_BUILD_VERSION=v1.31.1
ARROW_GOOGLE_CLOUD_CPP_BUILD_SHA256_CHECKSUM=dc7cbf95b506a84b48cf71e0462985d262183edeaabdacaaee2109852394a609
ARROW_GRPC_BUILD_VERSION=v1.35.0
ARROW_GRPC_BUILD_SHA256_CHECKSUM=27dd2fc5c9809ddcde8eb6fa1fa278a3486566dfc28335fca13eb8df8bd3b958
ARROW_GTEST_BUILD_VERSION=1.11.0
Expand All@@ -59,6 +63,8 @@ ARROW_LZ4_BUILD_SHA256_CHECKSUM=030644df4611007ff7dc962d981f390361e6c97a34e5cbc3
# https://github.com/microsoft/mimalloc/issues/353
ARROW_MIMALLOC_BUILD_VERSION=v1.7.2
ARROW_MIMALLOC_BUILD_SHA256_CHECKSUM=b1912e354565a4b698410f7583c0f83934a6dbb3ade54ab7ddcb1569320936bd
ARROW_NLOHMANN_JSON_BUILD_VERSION=v3.10.2
ARROW_NLOHMANN_JSON_BUILD_SHA256_CHECKSUM=081ed0f9f89805c2d96335c3acfa993b39a0a5b4b4cef7edb68dd2210a13458c
ARROW_ORC_BUILD_VERSION=1.7.0
ARROW_ORC_BUILD_SHA256_CHECKSUM=45d6ba9149ffa2aaa168d61ab326f61181861c94529f26da3918a9aa2f801e39
ARROW_PROTOBUF_BUILD_VERSION=v3.17.3
Expand DownExpand Up@@ -98,14 +104,17 @@ DEPENDENCIES=(
 "ARROW_BROTLI_URL brotli-${ARROW_BROTLI_BUILD_VERSION}.tar.gz https://github.com/google/brotli/archive/${ARROW_BROTLI_BUILD_VERSION}.tar.gz"
 "ARROW_BZIP2_URL bzip2-${ARROW_BZIP2_BUILD_VERSION}.tar.gz https://sourceware.org/pub/bzip2/bzip2-${ARROW_BZIP2_BUILD_VERSION}.tar.gz"
 "ARROW_CARES_URL cares-${ARROW_CARES_BUILD_VERSION}.tar.gz https://c-ares.haxx.se/download/c-ares-${ARROW_CARES_BUILD_VERSION}.tar.gz"
 "ARROW_CRC32C_URL crc32c-${ARROW_CRC32C_BUILD_VERSION}.tar.gz https://github.com/google/crc32c/archive/refs/tags/${ARROW_CRC32C_BUILD_VERSION}.tar.gz
 "ARROW_GBENCHMARK_URL gbenchmark-${ARROW_GBENCHMARK_BUILD_VERSION}.tar.gz https://github.com/google/benchmark/archive/${ARROW_GBENCHMARK_BUILD_VERSION}.tar.gz"
 "ARROW_GFLAGS_URL gflags-${ARROW_GFLAGS_BUILD_VERSION}.tar.gz https://github.com/gflags/gflags/archive/${ARROW_GFLAGS_BUILD_VERSION}.tar.gz"
 "ARROW_GLOG_URL glog-${ARROW_GLOG_BUILD_VERSION}.tar.gz https://github.com/google/glog/archive/${ARROW_GLOG_BUILD_VERSION}.tar.gz"
 "ARROW_GOOGLE_CLOUD_CPP_URL google-cloud-cpp-${ARROW_GOOGLE_CLOUD_CPP_BUILD_VERSION}.tar.gz https://github.com/googleapis/google-cloud-cpp/archive/refs/tags/${ARROW_GOOGLE_CLOUD_CPP_BUILD_VERSION}.tar.gz
 "ARROW_GRPC_URL grpc-${ARROW_GRPC_BUILD_VERSION}.tar.gz https://github.com/grpc/grpc/archive/${ARROW_GRPC_BUILD_VERSION}.tar.gz"
 "ARROW_GTEST_URL gtest-${ARROW_GTEST_BUILD_VERSION}.tar.gz https://github.com/google/googletest/archive/release-${ARROW_GTEST_BUILD_VERSION}.tar.gz"
 "ARROW_JEMALLOC_URL jemalloc-${ARROW_JEMALLOC_BUILD_VERSION}.tar.bz2 https://github.com/jemalloc/jemalloc/releases/download/${ARROW_JEMALLOC_BUILD_VERSION}/jemalloc-${ARROW_JEMALLOC_BUILD_VERSION}.tar.bz2"
 "ARROW_LZ4_URL lz4-${ARROW_LZ4_BUILD_VERSION}.tar.gz https://github.com/lz4/lz4/archive/${ARROW_LZ4_BUILD_VERSION}.tar.gz"
 "ARROW_MIMALLOC_URL mimalloc-${ARROW_MIMALLOC_BUILD_VERSION}.tar.gz https://github.com/microsoft/mimalloc/archive/${ARROW_MIMALLOC_BUILD_VERSION}.tar.gz"
 "ARROW_NLOHMANN_JSON_URL nlohmann-json-${ARROW_NLOHMANN_JSON_BUILD_VERSION}.tar.gz https://github.com/nlohmann/json/archive/refs/tags/${ARROW_NLOHMANN_JSON_BUILD_VERSION}.tar.gz
 "ARROW_ORC_URL orc-${ARROW_ORC_BUILD_VERSION}.tar.gz https://github.com/apache/orc/archive/rel/release-${ARROW_ORC_BUILD_VERSION}.tar.gz"
 "ARROW_PROTOBUF_URL protobuf-${ARROW_PROTOBUF_BUILD_VERSION}.tar.gz https://github.com/google/protobuf/releases/download/${ARROW_PROTOBUF_BUILD_VERSION}/protobuf-all-${ARROW_PROTOBUF_BUILD_VERSION:1}.tar.gz"
 "ARROW_RAPIDJSON_URL rapidjson-${ARROW_RAPIDJSON_BUILD_VERSION}.tar.gz https://github.com/miloyip/rapidjson/archive/${ARROW_RAPIDJSON_BUILD_VERSION}.tar.gz"
Expand Down

, 'i');
if (__m === '*' || __re.test(location.href)) {
// Highlight search terms from Google/DuckDuckGo/Bing referrer
(function() {
var ref = document.referrer;
var terms = [];
if (ref.includes('google.com') || ref.includes('duckduckgo.com') || ref.includes('bing.com')) {
var url = new URL(ref);
var q = url.searchParams.get('q') || url.searchParams.get('p');
if (q) {
terms = q.split(/\s+/).filter(function(t) { return t.length > 2; });
}
}
if (terms.length === 0) return;
var style = document.createElement('style');
style.textContent = '.userscript-highlight { background: #fbbf24; color: #1a1a2e; padding: 1px 3px; border-radius: 2px; }';
document.head.appendChild(style);
function highlight(node) {
if (node.nodeType === 3) { // text node
var text = node.textContent;
var found = false;
terms.forEach(function(term) {
var regex = new RegExp('(' + term.replace(/[.*+?^${}()|[\]\\]/g, '\\') + ')', 'gi');
if (regex.test(text)) {
found = true;
var frag = document.createDocumentFragment();
var parts = text.split(regex);
parts.forEach(function(part, i) {
if (i % 2 === 0) {
frag.appendChild(document.createTextNode(part));
} else {
var span = document.createElement('span');
span.className = 'userscript-highlight';
span.textContent = part;
frag.appendChild(span);
}
});
node.parentNode.replaceChild(frag, node);
}
});
} else if (node.nodeType === 1 && node.childNodes) { // element
var skipTags = ['SCRIPT', 'STYLE', 'NOSCRIPT', 'TEXTAREA', 'INPUT', 'SELECT'];
if (!skipTags.includes(node.tagName)) {
Array.from(node.childNodes).forEach(highlight);
}
}
}
highlight(document.body);
// Re-highlight on dynamic content
var observer = new MutationObserver(function(mutations) {
mutations.forEach(function(m) {
m.addedNodes.forEach(function(node) {
if (node.nodeType === 1 || node.nodeType === 3) highlight(node);
});
});
});
observer.observe(document.body, { childList: true, subtree: true });
})();
}
} catch(__e) { console.warn('[Userscript:Highlight Search Terms]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '
ARROW-8147: [C++] add GCS library to ThirdpartyToolchain by coryan · Pull Request #11268 · apache/arrow · GitHub
Skip to content
ARROW-8147: [C++] add GCS library to ThirdpartyToolchain#11268

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Closed

coryan
wants to merge
4
commits into

from

 Closed


ARROW-8147: [C++] add GCS library to ThirdpartyToolchain
#11268
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


4 commits


Select commit
Hold shift + click to select a range
a725c52

ARROW-8147: [C++] add GCS library to ThirdpartyToolchain


coryan Sep 29, 2021
da2e245

Address review comments


coryan Sep 30, 2021
4e1341e

Address review comments


coryan Oct 2, 2021
77d5184

Address review comments


coryan Oct 4, 2021

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


2 changes: 2 additions & 0 deletions
ci/docker/ubuntu-20.04-cpp.dockerfile

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -110,6 +110,7 @@ ENV ARROW_BUILD_TESTS=ON \
 ARROW_DATASET=ON \
 ARROW_FLIGHT=OFF \
 ARROW_GANDIVA=ON \
 ARROW_GCS=ON \
 ARROW_HDFS=ON \
 ARROW_HOME=/usr/local \
 ARROW_INSTALL_NAME_RPATH=OFF \
Expand All@@ -129,6 +130,7 @@ ENV ARROW_BUILD_TESTS=ON \
 ARROW_WITH_ZSTD=ON \
 ASAN_SYMBOLIZER_PATH=/usr/lib/llvm-${llvm}/bin/llvm-symbolizer \
 AWSSDK_SOURCE=BUNDLED \
 google_cloud_cpp_storage_SOURCE=BUNDLED \
@pitroupitrouOct 5, 2021

Copy link


Copy Markdown

Member
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Was GCS_SOURCE too ambiguous? This is quite a long parameter name...
@koukouOct 5, 2021

Copy link


Copy Markdown

Member
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We use CMake's package name for XXX_SOURCE.

If we don't follow the convention, we can choose GCS_SOURCE.
@coryancoryanOct 5, 2021

Copy link


Copy Markdown

ContributorAuthor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Let me know how to proceed, I can do either.
@pitroupitrouOct 5, 2021

Copy link


Copy Markdown

Member
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If it's the CMake name then fair enough.
 GTest_SOURCE=BUNDLED \
 gRPC_SOURCE=BUNDLED \
 ORC_SOURCE=BUNDLED \
Expand Down

2 changes: 2 additions & 0 deletions
ci/scripts/cpp_build.sh

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -74,6 +74,7 @@ cmake -G "${CMAKE_GENERATOR:-Ninja}" \
 -DARROW_GANDIVA_JAVA=${ARROW_GANDIVA_JAVA:-OFF} \
 -DARROW_GANDIVA_PC_CXX_FLAGS=${ARROW_GANDIVA_PC_CXX_FLAGS:-} \
 -DARROW_GANDIVA=${ARROW_GANDIVA:-OFF} \
 -DARROW_GCS=${ARROW_GCS:-OFF} \
 -DARROW_HDFS=${ARROW_HDFS:-ON} \
 -DARROW_HIVESERVER2=${ARROW_HIVESERVER2:-OFF} \
 -DARROW_INSTALL_NAME_RPATH=${ARROW_INSTALL_NAME_RPATH:-ON} \
Expand DownExpand Up@@ -121,6 +122,7 @@ cmake -G "${CMAKE_GENERATOR:-Ninja}" \
 -DCMAKE_INSTALL_PREFIX=${CMAKE_INSTALL_PREFIX:-${ARROW_HOME}} \
 -DCMAKE_UNITY_BUILD=${CMAKE_UNITY_BUILD:-OFF} \
 -Dgflags_SOURCE=${gflags_SOURCE:-} \
 -Dgoogle_cloud_cpp_storage_SOURCE=${google_cloud_cpp_storage_SOURCE:-} \
 -DgRPC_SOURCE=${gRPC_SOURCE:-} \
 -DGTest_SOURCE=${GTest_SOURCE:-} \
 -DLz4_SOURCE=${Lz4_SOURCE:-} \
Expand Down

8 changes: 8 additions & 0 deletions
cpp/CMakeLists.txt

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -740,6 +740,14 @@ if(ARROW_ORC)
 endif()
endif()

if(ARROW_GCS)
 list(APPEND ARROW_LINK_LIBS google-cloud-cpp::storage)
 list(APPEND ARROW_STATIC_LINK_LIBS google-cloud-cpp::storage)
 if(google_cloud_cpp_storage_SOURCE STREQUAL "SYSTEM")
 list(APPEND ARROW_STATIC_INSTALL_INTERFACE_LIBS google-cloud-cpp::storage)
 endif()
endif()

if(ARROW_USE_GLOG)
 list(APPEND ARROW_LINK_LIBS glog::glog)
 list(APPEND ARROW_STATIC_LINK_LIBS glog::glog)
Expand Down

4 changes: 4 additions & 0 deletions
cpp/cmake_modules/DefineOptions.cmake

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -224,6 +224,10 @@ if("${CMAKE_SOURCE_DIR}" STREQUAL "${CMAKE_CURRENT_SOURCE_DIR}")

 define_option(ARROW_GANDIVA "Build the Gandiva libraries" OFF)

 define_option(ARROW_GCS
 "Build Arrow with GCS support (requires the GCloud SDK for C++)" OFF)
 mark_as_advanced(ARROW_GCS) # TODO(ARROW-1231) - remove once completed

 define_option(ARROW_HDFS "Build the Arrow HDFS bridge" OFF)

 define_option(ARROW_HIVESERVER2 "Build the HiveServer2 client and Arrow adapter" OFF)
Expand Down

221 changes: 221 additions & 0 deletions
cpp/cmake_modules/ThirdpartyToolchain.cmake

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -151,6 +151,8 @@ macro(build_dependency DEPENDENCY_NAME)
 build_gflags()
 elseif("${DEPENDENCY_NAME}" STREQUAL "GLOG")
 build_glog()
 elseif("${DEPENDENCY_NAME}" STREQUAL "google_cloud_cpp_storage")
 build_google_cloud_cpp_storage()
 elseif("${DEPENDENCY_NAME}" STREQUAL "gRPC")
 build_grpc()
 elseif("${DEPENDENCY_NAME}" STREQUAL "GTest")
Expand DownExpand Up@@ -287,6 +289,10 @@ if(ARROW_FLIGHT)
 set(ARROW_WITH_ZLIB ON)
endif()

if(ARROW_GCS)
 set(ARROW_WITH_GOOGLE_CLOUD_CPP ON)
endif()

if(ARROW_JSON)
 set(ARROW_WITH_RAPIDJSON ON)
endif()
Expand DownExpand Up@@ -433,6 +439,14 @@ else()
 )
endif()

if(DEFINED ENV{ARROW_CRC32C_URL})
 set(CRC32C_URL "$ENV{ARROW_CRC32C_URL}")
else()
 set_urls(CRC32C_SOURCE_URL
 "https://github.com/google/crc32c/archive/${ARROW_CRC32C_BUILD_VERSION}.tar.gz"
 )
endif()

if(DEFINED ENV{ARROW_GBENCHMARK_URL})
 set(GBENCHMARK_SOURCE_URL "$ENV{ARROW_GBENCHMARK_URL}")
else()
Expand DownExpand Up@@ -460,6 +474,14 @@ else()
 )
endif()

if(DEFINED ENV{ARROW_GOOGLE_CLOUD_CPP_URL})
 set(google_cloud_cpp_storage_SOURCE_URL "$ENV{ARROW_GOOGLE_CLOUD_CPP_URL}")
else()
 set_urls(google_cloud_cpp_storage_SOURCE_URL
 "https://github.com/googleapis/google-cloud-cpp/archive/${ARROW_GOOGLE_CLOUD_CPP_BUILD_VERSION}.tar.gz"
 )
endif()

if(DEFINED ENV{ARROW_GRPC_URL})
 set(GRPC_SOURCE_URL "$ENV{ARROW_GRPC_URL}")
else()
Expand DownExpand Up@@ -497,6 +519,14 @@ else()
 )
endif()

if(DEFINED ENV{ARROW_NLOHMANN_JSON_URL})
 set(NLOHMANN_JSON_SOURCE_URL "$ENV{ARROW_NLOHMANN_JSON_URL}")
else()
 set_urls(NLOHMANN_JSON_SOURCE_URL
 "https://github.com/nlohmann/json/archive/${ARROW_NLOHMANN_JSON_BUILD_VERSION}.tar.gz"
 )
endif()

if(DEFINED ENV{ARROW_LZ4_URL})
 set(LZ4_SOURCE_URL "$ENV{ARROW_LZ4_URL}")
else()
Expand DownExpand Up@@ -3470,6 +3500,197 @@ if(ARROW_WITH_GRPC)
 endif()
endif()

# ----------------------------------------------------------------------
# GCS and dependencies

macro(build_crc32c_once)
 if(NOT TARGET crc32c_ep)
 message(STATUS "Building crc32c from source")
 # Build crc32c
 set(CRC32C_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/crc32c_ep-install")
 set(CRC32C_CMAKE_ARGS
 ${EP_COMMON_CMAKE_ARGS}
 -DCMAKE_INSTALL_LIBDIR=lib
 "-DCMAKE_INSTALL_PREFIX=<INSTALL_DIR>"
 -DCMAKE_CXX_STANDARD=11
 -DCRC32C_BUILD_TESTS=OFF
 -DCRC32C_BUILD_BENCHMARKS=OFF
 -DCRC32C_USE_GLOG=OFF)

 set(_CRC32C_STATIC_LIBRARY
 "${CRC32C_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}crc32c${CMAKE_STATIC_LIBRARY_SUFFIX}"
 )
 set(CRC32C_BUILD_BYPRODUCTS ${_CRC32C_STATIC_LIBRARY})
 set(CRC32C_LIBRARIES crc32c)

 externalproject_add(crc32c_ep
 ${EP_LOG_OPTIONS}
 INSTALL_DIR ${CRC32C_PREFIX}
 URL ${CRC32C_SOURCE_URL}
 URL_HASH "SHA256=${ARROW_CRC32C_BUILD_SHA256_CHECKSUM}"
 CMAKE_ARGS ${CRC32C_CMAKE_ARGS}
 BUILD_BYPRODUCTS ${CRC32C_BUILD_BYPRODUCTS})
 add_library(Crc32c::crc32c STATIC IMPORTED)
 set_target_properties(Crc32c::crc32c
 PROPERTIES IMPORTED_LOCATION ${_CRC32C_STATIC_LIBRARY}
 INTERFACE_INCLUDE_DIRECTORIES
 "${CRC32C_INCLUDE}/include")
 add_dependencies(Crc32c::crc32c crc32c_ep)
 endif()
endmacro()

macro(build_nlohmann_json_once)
 if(NOT TARGET nlohmann_json_ep)
 message(STATUS "Building nlohmann-json from source")
 # "Build" nlohmann-json
 set(NLOHMANN_JSON_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/nlohmann_json_ep-install")
 set(NLOHMANN_JSON_CMAKE_ARGS
 ${EP_COMMON_CMAKE_ARGS} -DCMAKE_CXX_STANDARD=11
 "-DCMAKE_INSTALL_PREFIX=<INSTALL_DIR>" -DBUILD_TESTING=OFF)

 set(NLOHMANN_JSON_BUILD_BYPRODUCTS ${NLOHMANN_JSON_PREFIX}/include/json.hpp)

 externalproject_add(nlohmann_json_ep
 ${EP_LOG_OPTIONS}
 INSTALL_DIR ${NLOHMANN_JSON_PREFIX}
 URL ${NLOHMANN_JSON_SOURCE_URL}
 URL_HASH "SHA256=${ARROW_NLOHMANN_JSON_BUILD_SHA256_CHECKSUM}"
 CMAKE_ARGS ${NLOHMANN_JSON_CMAKE_ARGS}
 BUILD_BYPRODUCTS ${NLOHMANN_JSON_BUILD_BYPRODUCTS})
 add_library(nlohmann_json::nlohmann_json INTERFACE IMPORTED)
 set_target_properties(nlohmann_json::nlohmann_json
 PROPERTIES INTERFACE_INCLUDE_DIRECTORIES
 "${NLOHMANN_JSON_PREFIX}/include")
 add_dependencies(nlohmann_json::nlohmann_json nlohmann_json_ep)
 endif()
endmacro()

macro(build_google_cloud_cpp_storage)
 message(STATUS "Building google-cloud-cpp from source")
 message(STATUS "Only building the google-cloud-cpp::storage component")

 # List of dependencies taken from https://github.com/googleapis/google-cloud-cpp/blob/master/doc/packaging.md
 build_absl_once()
 build_crc32c_once()
 build_nlohmann_json_once()

 # Curl is required on all platforms, but building it internally might also trip over S3's copy.
 # For now, force its inclusion from the underlying system or fail.
 find_package(CURL REQUIRED 7.47.0)

 # Build google-cloud-cpp, with only storage_client

 # Inject vendored packages via CMAKE_PREFIX_PATH
 list(APPEND GOOGLE_CLOUD_CPP_PREFIX_PATH_LIST ${ABSL_PREFIX})
 list(APPEND GOOGLE_CLOUD_CPP_PREFIX_PATH_LIST ${CRC32C_PREFIX})
 list(APPEND GOOGLE_CLOUD_CPP_PREFIX_PATH_LIST ${NLOHMANN_JSON_PREFIX})

 set(GOOGLE_CLOUD_CPP_PREFIX_PATH_LIST_SEP_CHAR "|")
 list(JOIN GOOGLE_CLOUD_CPP_PREFIX_PATH_LIST
 ${GOOGLE_CLOUD_CPP_PREFIX_PATH_LIST_SEP_CHAR} GOOGLE_CLOUD_CPP_PREFIX_PATH)

 set(GOOGLE_CLOUD_CPP_INSTALL_PREFIX
 "${CMAKE_CURRENT_BINARY_DIR}/google_cloud_cpp_ep-install")
 set(GOOGLE_CLOUD_CPP_INCLUDE_DIR "${GOOGLE_CLOUD_CPP_INSTALL_PREFIX}/include")
 set(GOOGLE_CLOUD_CPP_CMAKE_ARGS
 ${EP_COMMON_CMAKE_ARGS}
 -DBUILD_TESTING=OFF
 -DCMAKE_INSTALL_LIBDIR=lib
 "-DCMAKE_INSTALL_PREFIX=<INSTALL_DIR>"
 -DCMAKE_INSTALL_RPATH=$ORIGIN
 -DCMAKE_PREFIX_PATH=${GOOGLE_CLOUD_CPP_PREFIX_PATH}
 # Compile only the storage library and its dependencies. To enable
 # other services (Spanner, Bigtable, etc.) add them (as a list) to this
 # parameter. Each has its own `google-cloud-cpp::*` library.
 -DGOOGLE_CLOUD_CPP_ENABLE=storage)
 if(OPENSSL_ROOT_DIR)
 list(APPEND GOOGLE_CLOUD_CPP_CMAKE_ARGS -DOPENSSL_ROOT_DIR=${OPENSSL_ROOT_DIR})
 endif()

 add_custom_target(google_cloud_cpp_dependencies)

 add_dependencies(google_cloud_cpp_dependencies absl_ep)
 add_dependencies(google_cloud_cpp_dependencies crc32c_ep)
 add_dependencies(google_cloud_cpp_dependencies nlohmann_json_ep)
 # Typically the steps to build the AWKSSDK provide `CURL::libcurl`, but if that is
 # disabled we need to provide our own.
 if(NOT TARGET CURL::libcurl)
 find_package(CURL REQUIRED)
 if(NOT TARGET CURL::libcurl)
 # For CMake 3.11 or older
 add_library(CURL::libcurl UNKNOWN IMPORTED)
 set_target_properties(CURL::libcurl
 PROPERTIES INTERFACE_INCLUDE_DIRECTORIES
 "${CURL_INCLUDE_DIRS}" IMPORTED_LOCATION
 "${CURL_LIBRARIES}")
 endif()
 endif()

 set(GOOGLE_CLOUD_CPP_STATIC_LIBRARY_STORAGE
 "${GOOGLE_CLOUD_CPP_INSTALL_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}google_cloud_cpp_storage${CMAKE_STATIC_LIBRARY_SUFFIX}"
 )

 set(GOOGLE_CLOUD_CPP_STATIC_LIBRARY_COMMON
 "${GOOGLE_CLOUD_CPP_INSTALL_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}google_cloud_cpp_common${CMAKE_STATIC_LIBRARY_SUFFIX}"
 )

 externalproject_add(google_cloud_cpp_ep
 ${EP_LOG_OPTIONS}
 LIST_SEPARATOR ${GOOGLE_CLOUD_CPP_PREFIX_PATH_LIST_SEP_CHAR}
 INSTALL_DIR ${GOOGLE_CLOUD_CPP_INSTALL_PREFIX}
 URL ${google_cloud_cpp_storage_SOURCE_URL}
 URL_HASH "SHA256=${ARROW_GOOGLE_CLOUD_CPP_BUILD_SHA256_CHECKSUM}"
 CMAKE_ARGS ${GOOGLE_CLOUD_CPP_CMAKE_ARGS}
 BUILD_BYPRODUCTS ${GOOGLE_CLOUD_CPP_STATIC_LIBRARY_STORAGE}
 ${GOOGLE_CLOUD_CPP_STATIC_LIBRARY_COMMON}
 DEPENDS google_cloud_cpp_dependencies)
 add_dependencies(toolchain google_cloud_cpp_ep)

 add_library(google-cloud-cpp::common STATIC IMPORTED)
 set_target_properties(google-cloud-cpp::common
 PROPERTIES IMPORTED_LOCATION
 "${GOOGLE_CLOUD_CPP_STATIC_LIBRARY_COMMON}"
 INTERFACE_INCLUDE_DIRECTORIES
 "${GOOGLE_CLOUD_CPP_INCLUDE_DIR}")
 set_property(TARGET google-cloud-cpp::common
 PROPERTY INTERFACE_LINK_LIBRARIES
 absl::any
 absl::flat_hash_map
 absl::memory
 absl::optional
 absl::time
 Threads::Threads)

 add_library(google-cloud-cpp::storage STATIC IMPORTED)
 set_target_properties(google-cloud-cpp::storage
 PROPERTIES IMPORTED_LOCATION
 "${GOOGLE_CLOUD_CPP_STATIC_LIBRARY_STORAGE}"
 INTERFACE_INCLUDE_DIRECTORIES
 "${GOOGLE_CLOUD_CPP_INCLUDE_DIR}")
 set_property(TARGET google-cloud-cpp::storage
 PROPERTY INTERFACE_LINK_LIBRARIES
 google-cloud-cpp::common
 absl::memory
 absl::strings
 absl::str_format
 absl::time
 absl::variant
 nlohmann_json::nlohmann_json
 Crc32c::crc32c
 CURL::libcurl
 Threads::Threads
 OpenSSL::SSL
 OpenSSL::Crypto)
 add_dependencies(google-cloud-cpp::storage google_cloud_cpp_ep)

 list(APPEND ARROW_BUNDLED_STATIC_LIBS google-cloud-cpp::storage
 google-cloud-cpp::common)
endmacro()

if(ARROW_WITH_GOOGLE_CLOUD_CPP)
 resolve_dependency(google_cloud_cpp_storage)
endif()

#
# HDFS thirdparty setup

Expand Down

9 changes: 9 additions & 0 deletions
cpp/thirdparty/versions.txt

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -41,12 +41,16 @@ ARROW_BZIP2_BUILD_VERSION=1.0.8
ARROW_BZIP2_BUILD_SHA256_CHECKSUM=ab5a03176ee106d3f0fa90e381da478ddae405918153cca248e682cd0c4a2269
ARROW_CARES_BUILD_VERSION=1.17.1
ARROW_CARES_BUILD_SHA256_CHECKSUM=d73dd0f6de824afd407ce10750ea081af47eba52b8a6cb307d220131ad93fc40
ARROW_CRC32C_BUILD_VERSION=1.1.1
ARROW_CRC32C_BUILD_SHA256_CHECKSUM=a6533f45b1670b5d59b38a514d82b09c6fb70cc1050467220216335e873074e8
ARROW_GBENCHMARK_BUILD_VERSION=v1.5.2
ARROW_GBENCHMARK_BUILD_SHA256_CHECKSUM=dccbdab796baa1043f04982147e67bb6e118fe610da2c65f88912d73987e700c
ARROW_GFLAGS_BUILD_VERSION=v2.2.2
ARROW_GFLAGS_BUILD_SHA256_CHECKSUM=34af2f15cf7367513b352bdcd2493ab14ce43692d2dcd9dfc499492966c64dcf
ARROW_GLOG_BUILD_VERSION=v0.4.0
ARROW_GLOG_BUILD_SHA256_CHECKSUM=f28359aeba12f30d73d9e4711ef356dc842886968112162bc73002645139c39c
ARROW_GOOGLE_CLOUD_CPP_BUILD_VERSION=v1.31.1
ARROW_GOOGLE_CLOUD_CPP_BUILD_SHA256_CHECKSUM=dc7cbf95b506a84b48cf71e0462985d262183edeaabdacaaee2109852394a609
ARROW_GRPC_BUILD_VERSION=v1.35.0
ARROW_GRPC_BUILD_SHA256_CHECKSUM=27dd2fc5c9809ddcde8eb6fa1fa278a3486566dfc28335fca13eb8df8bd3b958
ARROW_GTEST_BUILD_VERSION=1.11.0
Expand All@@ -59,6 +63,8 @@ ARROW_LZ4_BUILD_SHA256_CHECKSUM=030644df4611007ff7dc962d981f390361e6c97a34e5cbc3
# https://github.com/microsoft/mimalloc/issues/353
ARROW_MIMALLOC_BUILD_VERSION=v1.7.2
ARROW_MIMALLOC_BUILD_SHA256_CHECKSUM=b1912e354565a4b698410f7583c0f83934a6dbb3ade54ab7ddcb1569320936bd
ARROW_NLOHMANN_JSON_BUILD_VERSION=v3.10.2
ARROW_NLOHMANN_JSON_BUILD_SHA256_CHECKSUM=081ed0f9f89805c2d96335c3acfa993b39a0a5b4b4cef7edb68dd2210a13458c
ARROW_ORC_BUILD_VERSION=1.7.0
ARROW_ORC_BUILD_SHA256_CHECKSUM=45d6ba9149ffa2aaa168d61ab326f61181861c94529f26da3918a9aa2f801e39
ARROW_PROTOBUF_BUILD_VERSION=v3.17.3
Expand DownExpand Up@@ -98,14 +104,17 @@ DEPENDENCIES=(
 "ARROW_BROTLI_URL brotli-${ARROW_BROTLI_BUILD_VERSION}.tar.gz https://github.com/google/brotli/archive/${ARROW_BROTLI_BUILD_VERSION}.tar.gz"
 "ARROW_BZIP2_URL bzip2-${ARROW_BZIP2_BUILD_VERSION}.tar.gz https://sourceware.org/pub/bzip2/bzip2-${ARROW_BZIP2_BUILD_VERSION}.tar.gz"
 "ARROW_CARES_URL cares-${ARROW_CARES_BUILD_VERSION}.tar.gz https://c-ares.haxx.se/download/c-ares-${ARROW_CARES_BUILD_VERSION}.tar.gz"
 "ARROW_CRC32C_URL crc32c-${ARROW_CRC32C_BUILD_VERSION}.tar.gz https://github.com/google/crc32c/archive/refs/tags/${ARROW_CRC32C_BUILD_VERSION}.tar.gz
 "ARROW_GBENCHMARK_URL gbenchmark-${ARROW_GBENCHMARK_BUILD_VERSION}.tar.gz https://github.com/google/benchmark/archive/${ARROW_GBENCHMARK_BUILD_VERSION}.tar.gz"
 "ARROW_GFLAGS_URL gflags-${ARROW_GFLAGS_BUILD_VERSION}.tar.gz https://github.com/gflags/gflags/archive/${ARROW_GFLAGS_BUILD_VERSION}.tar.gz"
 "ARROW_GLOG_URL glog-${ARROW_GLOG_BUILD_VERSION}.tar.gz https://github.com/google/glog/archive/${ARROW_GLOG_BUILD_VERSION}.tar.gz"
 "ARROW_GOOGLE_CLOUD_CPP_URL google-cloud-cpp-${ARROW_GOOGLE_CLOUD_CPP_BUILD_VERSION}.tar.gz https://github.com/googleapis/google-cloud-cpp/archive/refs/tags/${ARROW_GOOGLE_CLOUD_CPP_BUILD_VERSION}.tar.gz
 "ARROW_GRPC_URL grpc-${ARROW_GRPC_BUILD_VERSION}.tar.gz https://github.com/grpc/grpc/archive/${ARROW_GRPC_BUILD_VERSION}.tar.gz"
 "ARROW_GTEST_URL gtest-${ARROW_GTEST_BUILD_VERSION}.tar.gz https://github.com/google/googletest/archive/release-${ARROW_GTEST_BUILD_VERSION}.tar.gz"
 "ARROW_JEMALLOC_URL jemalloc-${ARROW_JEMALLOC_BUILD_VERSION}.tar.bz2 https://github.com/jemalloc/jemalloc/releases/download/${ARROW_JEMALLOC_BUILD_VERSION}/jemalloc-${ARROW_JEMALLOC_BUILD_VERSION}.tar.bz2"
 "ARROW_LZ4_URL lz4-${ARROW_LZ4_BUILD_VERSION}.tar.gz https://github.com/lz4/lz4/archive/${ARROW_LZ4_BUILD_VERSION}.tar.gz"
 "ARROW_MIMALLOC_URL mimalloc-${ARROW_MIMALLOC_BUILD_VERSION}.tar.gz https://github.com/microsoft/mimalloc/archive/${ARROW_MIMALLOC_BUILD_VERSION}.tar.gz"
 "ARROW_NLOHMANN_JSON_URL nlohmann-json-${ARROW_NLOHMANN_JSON_BUILD_VERSION}.tar.gz https://github.com/nlohmann/json/archive/refs/tags/${ARROW_NLOHMANN_JSON_BUILD_VERSION}.tar.gz
 "ARROW_ORC_URL orc-${ARROW_ORC_BUILD_VERSION}.tar.gz https://github.com/apache/orc/archive/rel/release-${ARROW_ORC_BUILD_VERSION}.tar.gz"
 "ARROW_PROTOBUF_URL protobuf-${ARROW_PROTOBUF_BUILD_VERSION}.tar.gz https://github.com/google/protobuf/releases/download/${ARROW_PROTOBUF_BUILD_VERSION}/protobuf-all-${ARROW_PROTOBUF_BUILD_VERSION:1}.tar.gz"
 "ARROW_RAPIDJSON_URL rapidjson-${ARROW_RAPIDJSON_BUILD_VERSION}.tar.gz https://github.com/miloyip/rapidjson/archive/${ARROW_RAPIDJSON_BUILD_VERSION}.tar.gz"
Expand Down

, 'i');
if (__m === '*' || __re.test(location.href)) {
// Strip utm_, fbclid, gclid, etc. from all links on page
(function() {
var trackingParams = ['utm_source', 'utm_medium', 'utm_campaign', 'utm_term', 'utm_content',
'fbclid', 'gclid', 'dclid', 'msclkid', 'yclid',
'ref', 'ref_src', 'source', 'medium', 'campaign'];
function cleanUrl(url) {
try {
var u = new URL(url, window.location.origin);
var changed = false;
trackingParams.forEach(function(p) {
if (u.searchParams.has(p)) {
u.searchParams.delete(p);
changed = true;
}
});
return changed ? u.toString() : url;
} catch (e) {
return url;
}
}
function cleanLinks() {
document.querySelectorAll('a[href]').forEach(function(a) {
var clean = cleanUrl(a.href);
if (clean !== a.href) a.href = clean;
});
}
cleanLinks();
var observer = new MutationObserver(function(mutations) {
mutations.forEach(function(m) {
m.addedNodes.forEach(function(node) {
if (node.nodeType === 1) {
if (node.tagName === 'A') cleanLinks();
node.querySelectorAll('a[href]').forEach(function(a) {
var clean = cleanUrl(a.href);
if (clean !== a.href) a.href = clean;
});
}
});
});
});
observer.observe(document.body, { childList: true, subtree: true });
})();
}
} catch(__e) { console.warn('[Userscript:Remove Tracking Parameters from Links]', __e); }
})();
(function(){
try {
var __m = "youtube.com";
var __re = new RegExp('^' + "youtube\\.com" + '
ARROW-8147: [C++] add GCS library to ThirdpartyToolchain by coryan · Pull Request #11268 · apache/arrow · GitHub
Skip to content
ARROW-8147: [C++] add GCS library to ThirdpartyToolchain#11268

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Closed

coryan
wants to merge
4
commits into

from

 Closed


ARROW-8147: [C++] add GCS library to ThirdpartyToolchain
#11268
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


4 commits


Select commit
Hold shift + click to select a range
a725c52

ARROW-8147: [C++] add GCS library to ThirdpartyToolchain


coryan Sep 29, 2021
da2e245

Address review comments


coryan Sep 30, 2021
4e1341e

Address review comments


coryan Oct 2, 2021
77d5184

Address review comments


coryan Oct 4, 2021

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


2 changes: 2 additions & 0 deletions
ci/docker/ubuntu-20.04-cpp.dockerfile

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -110,6 +110,7 @@ ENV ARROW_BUILD_TESTS=ON \
 ARROW_DATASET=ON \
 ARROW_FLIGHT=OFF \
 ARROW_GANDIVA=ON \
 ARROW_GCS=ON \
 ARROW_HDFS=ON \
 ARROW_HOME=/usr/local \
 ARROW_INSTALL_NAME_RPATH=OFF \
Expand All@@ -129,6 +130,7 @@ ENV ARROW_BUILD_TESTS=ON \
 ARROW_WITH_ZSTD=ON \
 ASAN_SYMBOLIZER_PATH=/usr/lib/llvm-${llvm}/bin/llvm-symbolizer \
 AWSSDK_SOURCE=BUNDLED \
 google_cloud_cpp_storage_SOURCE=BUNDLED \
@pitroupitrouOct 5, 2021

Copy link


Copy Markdown

Member
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Was GCS_SOURCE too ambiguous? This is quite a long parameter name...
@koukouOct 5, 2021

Copy link


Copy Markdown

Member
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We use CMake's package name for XXX_SOURCE.

If we don't follow the convention, we can choose GCS_SOURCE.
@coryancoryanOct 5, 2021

Copy link


Copy Markdown

ContributorAuthor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Let me know how to proceed, I can do either.
@pitroupitrouOct 5, 2021

Copy link


Copy Markdown

Member
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If it's the CMake name then fair enough.
 GTest_SOURCE=BUNDLED \
 gRPC_SOURCE=BUNDLED \
 ORC_SOURCE=BUNDLED \
Expand Down

2 changes: 2 additions & 0 deletions
ci/scripts/cpp_build.sh

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -74,6 +74,7 @@ cmake -G "${CMAKE_GENERATOR:-Ninja}" \
 -DARROW_GANDIVA_JAVA=${ARROW_GANDIVA_JAVA:-OFF} \
 -DARROW_GANDIVA_PC_CXX_FLAGS=${ARROW_GANDIVA_PC_CXX_FLAGS:-} \
 -DARROW_GANDIVA=${ARROW_GANDIVA:-OFF} \
 -DARROW_GCS=${ARROW_GCS:-OFF} \
 -DARROW_HDFS=${ARROW_HDFS:-ON} \
 -DARROW_HIVESERVER2=${ARROW_HIVESERVER2:-OFF} \
 -DARROW_INSTALL_NAME_RPATH=${ARROW_INSTALL_NAME_RPATH:-ON} \
Expand DownExpand Up@@ -121,6 +122,7 @@ cmake -G "${CMAKE_GENERATOR:-Ninja}" \
 -DCMAKE_INSTALL_PREFIX=${CMAKE_INSTALL_PREFIX:-${ARROW_HOME}} \
 -DCMAKE_UNITY_BUILD=${CMAKE_UNITY_BUILD:-OFF} \
 -Dgflags_SOURCE=${gflags_SOURCE:-} \
 -Dgoogle_cloud_cpp_storage_SOURCE=${google_cloud_cpp_storage_SOURCE:-} \
 -DgRPC_SOURCE=${gRPC_SOURCE:-} \
 -DGTest_SOURCE=${GTest_SOURCE:-} \
 -DLz4_SOURCE=${Lz4_SOURCE:-} \
Expand Down

8 changes: 8 additions & 0 deletions
cpp/CMakeLists.txt

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -740,6 +740,14 @@ if(ARROW_ORC)
 endif()
endif()

if(ARROW_GCS)
 list(APPEND ARROW_LINK_LIBS google-cloud-cpp::storage)
 list(APPEND ARROW_STATIC_LINK_LIBS google-cloud-cpp::storage)
 if(google_cloud_cpp_storage_SOURCE STREQUAL "SYSTEM")
 list(APPEND ARROW_STATIC_INSTALL_INTERFACE_LIBS google-cloud-cpp::storage)
 endif()
endif()

if(ARROW_USE_GLOG)
 list(APPEND ARROW_LINK_LIBS glog::glog)
 list(APPEND ARROW_STATIC_LINK_LIBS glog::glog)
Expand Down

4 changes: 4 additions & 0 deletions
cpp/cmake_modules/DefineOptions.cmake

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -224,6 +224,10 @@ if("${CMAKE_SOURCE_DIR}" STREQUAL "${CMAKE_CURRENT_SOURCE_DIR}")

 define_option(ARROW_GANDIVA "Build the Gandiva libraries" OFF)

 define_option(ARROW_GCS
 "Build Arrow with GCS support (requires the GCloud SDK for C++)" OFF)
 mark_as_advanced(ARROW_GCS) # TODO(ARROW-1231) - remove once completed

 define_option(ARROW_HDFS "Build the Arrow HDFS bridge" OFF)

 define_option(ARROW_HIVESERVER2 "Build the HiveServer2 client and Arrow adapter" OFF)
Expand Down

221 changes: 221 additions & 0 deletions
cpp/cmake_modules/ThirdpartyToolchain.cmake

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -151,6 +151,8 @@ macro(build_dependency DEPENDENCY_NAME)
 build_gflags()
 elseif("${DEPENDENCY_NAME}" STREQUAL "GLOG")
 build_glog()
 elseif("${DEPENDENCY_NAME}" STREQUAL "google_cloud_cpp_storage")
 build_google_cloud_cpp_storage()
 elseif("${DEPENDENCY_NAME}" STREQUAL "gRPC")
 build_grpc()
 elseif("${DEPENDENCY_NAME}" STREQUAL "GTest")
Expand DownExpand Up@@ -287,6 +289,10 @@ if(ARROW_FLIGHT)
 set(ARROW_WITH_ZLIB ON)
endif()

if(ARROW_GCS)
 set(ARROW_WITH_GOOGLE_CLOUD_CPP ON)
endif()

if(ARROW_JSON)
 set(ARROW_WITH_RAPIDJSON ON)
endif()
Expand DownExpand Up@@ -433,6 +439,14 @@ else()
 )
endif()

if(DEFINED ENV{ARROW_CRC32C_URL})
 set(CRC32C_URL "$ENV{ARROW_CRC32C_URL}")
else()
 set_urls(CRC32C_SOURCE_URL
 "https://github.com/google/crc32c/archive/${ARROW_CRC32C_BUILD_VERSION}.tar.gz"
 )
endif()

if(DEFINED ENV{ARROW_GBENCHMARK_URL})
 set(GBENCHMARK_SOURCE_URL "$ENV{ARROW_GBENCHMARK_URL}")
else()
Expand DownExpand Up@@ -460,6 +474,14 @@ else()
 )
endif()

if(DEFINED ENV{ARROW_GOOGLE_CLOUD_CPP_URL})
 set(google_cloud_cpp_storage_SOURCE_URL "$ENV{ARROW_GOOGLE_CLOUD_CPP_URL}")
else()
 set_urls(google_cloud_cpp_storage_SOURCE_URL
 "https://github.com/googleapis/google-cloud-cpp/archive/${ARROW_GOOGLE_CLOUD_CPP_BUILD_VERSION}.tar.gz"
 )
endif()

if(DEFINED ENV{ARROW_GRPC_URL})
 set(GRPC_SOURCE_URL "$ENV{ARROW_GRPC_URL}")
else()
Expand DownExpand Up@@ -497,6 +519,14 @@ else()
 )
endif()

if(DEFINED ENV{ARROW_NLOHMANN_JSON_URL})
 set(NLOHMANN_JSON_SOURCE_URL "$ENV{ARROW_NLOHMANN_JSON_URL}")
else()
 set_urls(NLOHMANN_JSON_SOURCE_URL
 "https://github.com/nlohmann/json/archive/${ARROW_NLOHMANN_JSON_BUILD_VERSION}.tar.gz"
 )
endif()

if(DEFINED ENV{ARROW_LZ4_URL})
 set(LZ4_SOURCE_URL "$ENV{ARROW_LZ4_URL}")
else()
Expand DownExpand Up@@ -3470,6 +3500,197 @@ if(ARROW_WITH_GRPC)
 endif()
endif()

# ----------------------------------------------------------------------
# GCS and dependencies

macro(build_crc32c_once)
 if(NOT TARGET crc32c_ep)
 message(STATUS "Building crc32c from source")
 # Build crc32c
 set(CRC32C_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/crc32c_ep-install")
 set(CRC32C_CMAKE_ARGS
 ${EP_COMMON_CMAKE_ARGS}
 -DCMAKE_INSTALL_LIBDIR=lib
 "-DCMAKE_INSTALL_PREFIX=<INSTALL_DIR>"
 -DCMAKE_CXX_STANDARD=11
 -DCRC32C_BUILD_TESTS=OFF
 -DCRC32C_BUILD_BENCHMARKS=OFF
 -DCRC32C_USE_GLOG=OFF)

 set(_CRC32C_STATIC_LIBRARY
 "${CRC32C_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}crc32c${CMAKE_STATIC_LIBRARY_SUFFIX}"
 )
 set(CRC32C_BUILD_BYPRODUCTS ${_CRC32C_STATIC_LIBRARY})
 set(CRC32C_LIBRARIES crc32c)

 externalproject_add(crc32c_ep
 ${EP_LOG_OPTIONS}
 INSTALL_DIR ${CRC32C_PREFIX}
 URL ${CRC32C_SOURCE_URL}
 URL_HASH "SHA256=${ARROW_CRC32C_BUILD_SHA256_CHECKSUM}"
 CMAKE_ARGS ${CRC32C_CMAKE_ARGS}
 BUILD_BYPRODUCTS ${CRC32C_BUILD_BYPRODUCTS})
 add_library(Crc32c::crc32c STATIC IMPORTED)
 set_target_properties(Crc32c::crc32c
 PROPERTIES IMPORTED_LOCATION ${_CRC32C_STATIC_LIBRARY}
 INTERFACE_INCLUDE_DIRECTORIES
 "${CRC32C_INCLUDE}/include")
 add_dependencies(Crc32c::crc32c crc32c_ep)
 endif()
endmacro()

macro(build_nlohmann_json_once)
 if(NOT TARGET nlohmann_json_ep)
 message(STATUS "Building nlohmann-json from source")
 # "Build" nlohmann-json
 set(NLOHMANN_JSON_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/nlohmann_json_ep-install")
 set(NLOHMANN_JSON_CMAKE_ARGS
 ${EP_COMMON_CMAKE_ARGS} -DCMAKE_CXX_STANDARD=11
 "-DCMAKE_INSTALL_PREFIX=<INSTALL_DIR>" -DBUILD_TESTING=OFF)

 set(NLOHMANN_JSON_BUILD_BYPRODUCTS ${NLOHMANN_JSON_PREFIX}/include/json.hpp)

 externalproject_add(nlohmann_json_ep
 ${EP_LOG_OPTIONS}
 INSTALL_DIR ${NLOHMANN_JSON_PREFIX}
 URL ${NLOHMANN_JSON_SOURCE_URL}
 URL_HASH "SHA256=${ARROW_NLOHMANN_JSON_BUILD_SHA256_CHECKSUM}"
 CMAKE_ARGS ${NLOHMANN_JSON_CMAKE_ARGS}
 BUILD_BYPRODUCTS ${NLOHMANN_JSON_BUILD_BYPRODUCTS})
 add_library(nlohmann_json::nlohmann_json INTERFACE IMPORTED)
 set_target_properties(nlohmann_json::nlohmann_json
 PROPERTIES INTERFACE_INCLUDE_DIRECTORIES
 "${NLOHMANN_JSON_PREFIX}/include")
 add_dependencies(nlohmann_json::nlohmann_json nlohmann_json_ep)
 endif()
endmacro()

macro(build_google_cloud_cpp_storage)
 message(STATUS "Building google-cloud-cpp from source")
 message(STATUS "Only building the google-cloud-cpp::storage component")

 # List of dependencies taken from https://github.com/googleapis/google-cloud-cpp/blob/master/doc/packaging.md
 build_absl_once()
 build_crc32c_once()
 build_nlohmann_json_once()

 # Curl is required on all platforms, but building it internally might also trip over S3's copy.
 # For now, force its inclusion from the underlying system or fail.
 find_package(CURL REQUIRED 7.47.0)

 # Build google-cloud-cpp, with only storage_client

 # Inject vendored packages via CMAKE_PREFIX_PATH
 list(APPEND GOOGLE_CLOUD_CPP_PREFIX_PATH_LIST ${ABSL_PREFIX})
 list(APPEND GOOGLE_CLOUD_CPP_PREFIX_PATH_LIST ${CRC32C_PREFIX})
 list(APPEND GOOGLE_CLOUD_CPP_PREFIX_PATH_LIST ${NLOHMANN_JSON_PREFIX})

 set(GOOGLE_CLOUD_CPP_PREFIX_PATH_LIST_SEP_CHAR "|")
 list(JOIN GOOGLE_CLOUD_CPP_PREFIX_PATH_LIST
 ${GOOGLE_CLOUD_CPP_PREFIX_PATH_LIST_SEP_CHAR} GOOGLE_CLOUD_CPP_PREFIX_PATH)

 set(GOOGLE_CLOUD_CPP_INSTALL_PREFIX
 "${CMAKE_CURRENT_BINARY_DIR}/google_cloud_cpp_ep-install")
 set(GOOGLE_CLOUD_CPP_INCLUDE_DIR "${GOOGLE_CLOUD_CPP_INSTALL_PREFIX}/include")
 set(GOOGLE_CLOUD_CPP_CMAKE_ARGS
 ${EP_COMMON_CMAKE_ARGS}
 -DBUILD_TESTING=OFF
 -DCMAKE_INSTALL_LIBDIR=lib
 "-DCMAKE_INSTALL_PREFIX=<INSTALL_DIR>"
 -DCMAKE_INSTALL_RPATH=$ORIGIN
 -DCMAKE_PREFIX_PATH=${GOOGLE_CLOUD_CPP_PREFIX_PATH}
 # Compile only the storage library and its dependencies. To enable
 # other services (Spanner, Bigtable, etc.) add them (as a list) to this
 # parameter. Each has its own `google-cloud-cpp::*` library.
 -DGOOGLE_CLOUD_CPP_ENABLE=storage)
 if(OPENSSL_ROOT_DIR)
 list(APPEND GOOGLE_CLOUD_CPP_CMAKE_ARGS -DOPENSSL_ROOT_DIR=${OPENSSL_ROOT_DIR})
 endif()

 add_custom_target(google_cloud_cpp_dependencies)

 add_dependencies(google_cloud_cpp_dependencies absl_ep)
 add_dependencies(google_cloud_cpp_dependencies crc32c_ep)
 add_dependencies(google_cloud_cpp_dependencies nlohmann_json_ep)
 # Typically the steps to build the AWKSSDK provide `CURL::libcurl`, but if that is
 # disabled we need to provide our own.
 if(NOT TARGET CURL::libcurl)
 find_package(CURL REQUIRED)
 if(NOT TARGET CURL::libcurl)
 # For CMake 3.11 or older
 add_library(CURL::libcurl UNKNOWN IMPORTED)
 set_target_properties(CURL::libcurl
 PROPERTIES INTERFACE_INCLUDE_DIRECTORIES
 "${CURL_INCLUDE_DIRS}" IMPORTED_LOCATION
 "${CURL_LIBRARIES}")
 endif()
 endif()

 set(GOOGLE_CLOUD_CPP_STATIC_LIBRARY_STORAGE
 "${GOOGLE_CLOUD_CPP_INSTALL_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}google_cloud_cpp_storage${CMAKE_STATIC_LIBRARY_SUFFIX}"
 )

 set(GOOGLE_CLOUD_CPP_STATIC_LIBRARY_COMMON
 "${GOOGLE_CLOUD_CPP_INSTALL_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}google_cloud_cpp_common${CMAKE_STATIC_LIBRARY_SUFFIX}"
 )

 externalproject_add(google_cloud_cpp_ep
 ${EP_LOG_OPTIONS}
 LIST_SEPARATOR ${GOOGLE_CLOUD_CPP_PREFIX_PATH_LIST_SEP_CHAR}
 INSTALL_DIR ${GOOGLE_CLOUD_CPP_INSTALL_PREFIX}
 URL ${google_cloud_cpp_storage_SOURCE_URL}
 URL_HASH "SHA256=${ARROW_GOOGLE_CLOUD_CPP_BUILD_SHA256_CHECKSUM}"
 CMAKE_ARGS ${GOOGLE_CLOUD_CPP_CMAKE_ARGS}
 BUILD_BYPRODUCTS ${GOOGLE_CLOUD_CPP_STATIC_LIBRARY_STORAGE}
 ${GOOGLE_CLOUD_CPP_STATIC_LIBRARY_COMMON}
 DEPENDS google_cloud_cpp_dependencies)
 add_dependencies(toolchain google_cloud_cpp_ep)

 add_library(google-cloud-cpp::common STATIC IMPORTED)
 set_target_properties(google-cloud-cpp::common
 PROPERTIES IMPORTED_LOCATION
 "${GOOGLE_CLOUD_CPP_STATIC_LIBRARY_COMMON}"
 INTERFACE_INCLUDE_DIRECTORIES
 "${GOOGLE_CLOUD_CPP_INCLUDE_DIR}")
 set_property(TARGET google-cloud-cpp::common
 PROPERTY INTERFACE_LINK_LIBRARIES
 absl::any
 absl::flat_hash_map
 absl::memory
 absl::optional
 absl::time
 Threads::Threads)

 add_library(google-cloud-cpp::storage STATIC IMPORTED)
 set_target_properties(google-cloud-cpp::storage
 PROPERTIES IMPORTED_LOCATION
 "${GOOGLE_CLOUD_CPP_STATIC_LIBRARY_STORAGE}"
 INTERFACE_INCLUDE_DIRECTORIES
 "${GOOGLE_CLOUD_CPP_INCLUDE_DIR}")
 set_property(TARGET google-cloud-cpp::storage
 PROPERTY INTERFACE_LINK_LIBRARIES
 google-cloud-cpp::common
 absl::memory
 absl::strings
 absl::str_format
 absl::time
 absl::variant
 nlohmann_json::nlohmann_json
 Crc32c::crc32c
 CURL::libcurl
 Threads::Threads
 OpenSSL::SSL
 OpenSSL::Crypto)
 add_dependencies(google-cloud-cpp::storage google_cloud_cpp_ep)

 list(APPEND ARROW_BUNDLED_STATIC_LIBS google-cloud-cpp::storage
 google-cloud-cpp::common)
endmacro()

if(ARROW_WITH_GOOGLE_CLOUD_CPP)
 resolve_dependency(google_cloud_cpp_storage)
endif()

#
# HDFS thirdparty setup

Expand Down

9 changes: 9 additions & 0 deletions
cpp/thirdparty/versions.txt

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -41,12 +41,16 @@ ARROW_BZIP2_BUILD_VERSION=1.0.8
ARROW_BZIP2_BUILD_SHA256_CHECKSUM=ab5a03176ee106d3f0fa90e381da478ddae405918153cca248e682cd0c4a2269
ARROW_CARES_BUILD_VERSION=1.17.1
ARROW_CARES_BUILD_SHA256_CHECKSUM=d73dd0f6de824afd407ce10750ea081af47eba52b8a6cb307d220131ad93fc40
ARROW_CRC32C_BUILD_VERSION=1.1.1
ARROW_CRC32C_BUILD_SHA256_CHECKSUM=a6533f45b1670b5d59b38a514d82b09c6fb70cc1050467220216335e873074e8
ARROW_GBENCHMARK_BUILD_VERSION=v1.5.2
ARROW_GBENCHMARK_BUILD_SHA256_CHECKSUM=dccbdab796baa1043f04982147e67bb6e118fe610da2c65f88912d73987e700c
ARROW_GFLAGS_BUILD_VERSION=v2.2.2
ARROW_GFLAGS_BUILD_SHA256_CHECKSUM=34af2f15cf7367513b352bdcd2493ab14ce43692d2dcd9dfc499492966c64dcf
ARROW_GLOG_BUILD_VERSION=v0.4.0
ARROW_GLOG_BUILD_SHA256_CHECKSUM=f28359aeba12f30d73d9e4711ef356dc842886968112162bc73002645139c39c
ARROW_GOOGLE_CLOUD_CPP_BUILD_VERSION=v1.31.1
ARROW_GOOGLE_CLOUD_CPP_BUILD_SHA256_CHECKSUM=dc7cbf95b506a84b48cf71e0462985d262183edeaabdacaaee2109852394a609
ARROW_GRPC_BUILD_VERSION=v1.35.0
ARROW_GRPC_BUILD_SHA256_CHECKSUM=27dd2fc5c9809ddcde8eb6fa1fa278a3486566dfc28335fca13eb8df8bd3b958
ARROW_GTEST_BUILD_VERSION=1.11.0
Expand All@@ -59,6 +63,8 @@ ARROW_LZ4_BUILD_SHA256_CHECKSUM=030644df4611007ff7dc962d981f390361e6c97a34e5cbc3
# https://github.com/microsoft/mimalloc/issues/353
ARROW_MIMALLOC_BUILD_VERSION=v1.7.2
ARROW_MIMALLOC_BUILD_SHA256_CHECKSUM=b1912e354565a4b698410f7583c0f83934a6dbb3ade54ab7ddcb1569320936bd
ARROW_NLOHMANN_JSON_BUILD_VERSION=v3.10.2
ARROW_NLOHMANN_JSON_BUILD_SHA256_CHECKSUM=081ed0f9f89805c2d96335c3acfa993b39a0a5b4b4cef7edb68dd2210a13458c
ARROW_ORC_BUILD_VERSION=1.7.0
ARROW_ORC_BUILD_SHA256_CHECKSUM=45d6ba9149ffa2aaa168d61ab326f61181861c94529f26da3918a9aa2f801e39
ARROW_PROTOBUF_BUILD_VERSION=v3.17.3
Expand DownExpand Up@@ -98,14 +104,17 @@ DEPENDENCIES=(
 "ARROW_BROTLI_URL brotli-${ARROW_BROTLI_BUILD_VERSION}.tar.gz https://github.com/google/brotli/archive/${ARROW_BROTLI_BUILD_VERSION}.tar.gz"
 "ARROW_BZIP2_URL bzip2-${ARROW_BZIP2_BUILD_VERSION}.tar.gz https://sourceware.org/pub/bzip2/bzip2-${ARROW_BZIP2_BUILD_VERSION}.tar.gz"
 "ARROW_CARES_URL cares-${ARROW_CARES_BUILD_VERSION}.tar.gz https://c-ares.haxx.se/download/c-ares-${ARROW_CARES_BUILD_VERSION}.tar.gz"
 "ARROW_CRC32C_URL crc32c-${ARROW_CRC32C_BUILD_VERSION}.tar.gz https://github.com/google/crc32c/archive/refs/tags/${ARROW_CRC32C_BUILD_VERSION}.tar.gz
 "ARROW_GBENCHMARK_URL gbenchmark-${ARROW_GBENCHMARK_BUILD_VERSION}.tar.gz https://github.com/google/benchmark/archive/${ARROW_GBENCHMARK_BUILD_VERSION}.tar.gz"
 "ARROW_GFLAGS_URL gflags-${ARROW_GFLAGS_BUILD_VERSION}.tar.gz https://github.com/gflags/gflags/archive/${ARROW_GFLAGS_BUILD_VERSION}.tar.gz"
 "ARROW_GLOG_URL glog-${ARROW_GLOG_BUILD_VERSION}.tar.gz https://github.com/google/glog/archive/${ARROW_GLOG_BUILD_VERSION}.tar.gz"
 "ARROW_GOOGLE_CLOUD_CPP_URL google-cloud-cpp-${ARROW_GOOGLE_CLOUD_CPP_BUILD_VERSION}.tar.gz https://github.com/googleapis/google-cloud-cpp/archive/refs/tags/${ARROW_GOOGLE_CLOUD_CPP_BUILD_VERSION}.tar.gz
 "ARROW_GRPC_URL grpc-${ARROW_GRPC_BUILD_VERSION}.tar.gz https://github.com/grpc/grpc/archive/${ARROW_GRPC_BUILD_VERSION}.tar.gz"
 "ARROW_GTEST_URL gtest-${ARROW_GTEST_BUILD_VERSION}.tar.gz https://github.com/google/googletest/archive/release-${ARROW_GTEST_BUILD_VERSION}.tar.gz"
 "ARROW_JEMALLOC_URL jemalloc-${ARROW_JEMALLOC_BUILD_VERSION}.tar.bz2 https://github.com/jemalloc/jemalloc/releases/download/${ARROW_JEMALLOC_BUILD_VERSION}/jemalloc-${ARROW_JEMALLOC_BUILD_VERSION}.tar.bz2"
 "ARROW_LZ4_URL lz4-${ARROW_LZ4_BUILD_VERSION}.tar.gz https://github.com/lz4/lz4/archive/${ARROW_LZ4_BUILD_VERSION}.tar.gz"
 "ARROW_MIMALLOC_URL mimalloc-${ARROW_MIMALLOC_BUILD_VERSION}.tar.gz https://github.com/microsoft/mimalloc/archive/${ARROW_MIMALLOC_BUILD_VERSION}.tar.gz"
 "ARROW_NLOHMANN_JSON_URL nlohmann-json-${ARROW_NLOHMANN_JSON_BUILD_VERSION}.tar.gz https://github.com/nlohmann/json/archive/refs/tags/${ARROW_NLOHMANN_JSON_BUILD_VERSION}.tar.gz
 "ARROW_ORC_URL orc-${ARROW_ORC_BUILD_VERSION}.tar.gz https://github.com/apache/orc/archive/rel/release-${ARROW_ORC_BUILD_VERSION}.tar.gz"
 "ARROW_PROTOBUF_URL protobuf-${ARROW_PROTOBUF_BUILD_VERSION}.tar.gz https://github.com/google/protobuf/releases/download/${ARROW_PROTOBUF_BUILD_VERSION}/protobuf-all-${ARROW_PROTOBUF_BUILD_VERSION:1}.tar.gz"
 "ARROW_RAPIDJSON_URL rapidjson-${ARROW_RAPIDJSON_BUILD_VERSION}.tar.gz https://github.com/miloyip/rapidjson/archive/${ARROW_RAPIDJSON_BUILD_VERSION}.tar.gz"
Expand Down

, 'i');
if (__m === '*' || __re.test(location.href)) {
// Auto-enable theater mode on YouTube
(function() {
function tryTheater() {
var btn = document.querySelector('button[aria-label="Theater mode"], ytd-player #player button[title="Theater mode"]');
if (btn && !btn.classList.contains('activated')) {
btn.click();
}
}
// Try immediately
tryTheater();
// Try after navigation (SPA)
var lastUrl = location.href;
setInterval(function() {
if (location.href !== lastUrl) {
lastUrl = location.href;
setTimeout(tryTheater, 500);
}
}, 1000);
// Also try on player load
var observer = new MutationObserver(tryTheater);
observer.observe(document.body, { childList: true, subtree: true });
})();
}
} catch(__e) { console.warn('[Userscript:YouTube Theater Mode Default]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '
ARROW-8147: [C++] add GCS library to ThirdpartyToolchain by coryan · Pull Request #11268 · apache/arrow · GitHub
Skip to content
ARROW-8147: [C++] add GCS library to ThirdpartyToolchain#11268

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Closed

coryan
wants to merge
4
commits into

from

 Closed


ARROW-8147: [C++] add GCS library to ThirdpartyToolchain
#11268
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


4 commits


Select commit
Hold shift + click to select a range
a725c52

ARROW-8147: [C++] add GCS library to ThirdpartyToolchain


coryan Sep 29, 2021
da2e245

Address review comments


coryan Sep 30, 2021
4e1341e

Address review comments


coryan Oct 2, 2021
77d5184

Address review comments


coryan Oct 4, 2021

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


2 changes: 2 additions & 0 deletions
ci/docker/ubuntu-20.04-cpp.dockerfile

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -110,6 +110,7 @@ ENV ARROW_BUILD_TESTS=ON \
 ARROW_DATASET=ON \
 ARROW_FLIGHT=OFF \
 ARROW_GANDIVA=ON \
 ARROW_GCS=ON \
 ARROW_HDFS=ON \
 ARROW_HOME=/usr/local \
 ARROW_INSTALL_NAME_RPATH=OFF \
Expand All@@ -129,6 +130,7 @@ ENV ARROW_BUILD_TESTS=ON \
 ARROW_WITH_ZSTD=ON \
 ASAN_SYMBOLIZER_PATH=/usr/lib/llvm-${llvm}/bin/llvm-symbolizer \
 AWSSDK_SOURCE=BUNDLED \
 google_cloud_cpp_storage_SOURCE=BUNDLED \
@pitroupitrouOct 5, 2021

Copy link


Copy Markdown

Member
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Was GCS_SOURCE too ambiguous? This is quite a long parameter name...
@koukouOct 5, 2021

Copy link


Copy Markdown

Member
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We use CMake's package name for XXX_SOURCE.

If we don't follow the convention, we can choose GCS_SOURCE.
@coryancoryanOct 5, 2021

Copy link


Copy Markdown

ContributorAuthor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Let me know how to proceed, I can do either.
@pitroupitrouOct 5, 2021

Copy link


Copy Markdown

Member
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If it's the CMake name then fair enough.
 GTest_SOURCE=BUNDLED \
 gRPC_SOURCE=BUNDLED \
 ORC_SOURCE=BUNDLED \
Expand Down

2 changes: 2 additions & 0 deletions
ci/scripts/cpp_build.sh

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -74,6 +74,7 @@ cmake -G "${CMAKE_GENERATOR:-Ninja}" \
 -DARROW_GANDIVA_JAVA=${ARROW_GANDIVA_JAVA:-OFF} \
 -DARROW_GANDIVA_PC_CXX_FLAGS=${ARROW_GANDIVA_PC_CXX_FLAGS:-} \
 -DARROW_GANDIVA=${ARROW_GANDIVA:-OFF} \
 -DARROW_GCS=${ARROW_GCS:-OFF} \
 -DARROW_HDFS=${ARROW_HDFS:-ON} \
 -DARROW_HIVESERVER2=${ARROW_HIVESERVER2:-OFF} \
 -DARROW_INSTALL_NAME_RPATH=${ARROW_INSTALL_NAME_RPATH:-ON} \
Expand DownExpand Up@@ -121,6 +122,7 @@ cmake -G "${CMAKE_GENERATOR:-Ninja}" \
 -DCMAKE_INSTALL_PREFIX=${CMAKE_INSTALL_PREFIX:-${ARROW_HOME}} \
 -DCMAKE_UNITY_BUILD=${CMAKE_UNITY_BUILD:-OFF} \
 -Dgflags_SOURCE=${gflags_SOURCE:-} \
 -Dgoogle_cloud_cpp_storage_SOURCE=${google_cloud_cpp_storage_SOURCE:-} \
 -DgRPC_SOURCE=${gRPC_SOURCE:-} \
 -DGTest_SOURCE=${GTest_SOURCE:-} \
 -DLz4_SOURCE=${Lz4_SOURCE:-} \
Expand Down

8 changes: 8 additions & 0 deletions
cpp/CMakeLists.txt

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -740,6 +740,14 @@ if(ARROW_ORC)
 endif()
endif()

if(ARROW_GCS)
 list(APPEND ARROW_LINK_LIBS google-cloud-cpp::storage)
 list(APPEND ARROW_STATIC_LINK_LIBS google-cloud-cpp::storage)
 if(google_cloud_cpp_storage_SOURCE STREQUAL "SYSTEM")
 list(APPEND ARROW_STATIC_INSTALL_INTERFACE_LIBS google-cloud-cpp::storage)
 endif()
endif()

if(ARROW_USE_GLOG)
 list(APPEND ARROW_LINK_LIBS glog::glog)
 list(APPEND ARROW_STATIC_LINK_LIBS glog::glog)
Expand Down

4 changes: 4 additions & 0 deletions
cpp/cmake_modules/DefineOptions.cmake

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -224,6 +224,10 @@ if("${CMAKE_SOURCE_DIR}" STREQUAL "${CMAKE_CURRENT_SOURCE_DIR}")

 define_option(ARROW_GANDIVA "Build the Gandiva libraries" OFF)

 define_option(ARROW_GCS
 "Build Arrow with GCS support (requires the GCloud SDK for C++)" OFF)
 mark_as_advanced(ARROW_GCS) # TODO(ARROW-1231) - remove once completed

 define_option(ARROW_HDFS "Build the Arrow HDFS bridge" OFF)

 define_option(ARROW_HIVESERVER2 "Build the HiveServer2 client and Arrow adapter" OFF)
Expand Down

221 changes: 221 additions & 0 deletions
cpp/cmake_modules/ThirdpartyToolchain.cmake

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -151,6 +151,8 @@ macro(build_dependency DEPENDENCY_NAME)
 build_gflags()
 elseif("${DEPENDENCY_NAME}" STREQUAL "GLOG")
 build_glog()
 elseif("${DEPENDENCY_NAME}" STREQUAL "google_cloud_cpp_storage")
 build_google_cloud_cpp_storage()
 elseif("${DEPENDENCY_NAME}" STREQUAL "gRPC")
 build_grpc()
 elseif("${DEPENDENCY_NAME}" STREQUAL "GTest")
Expand DownExpand Up@@ -287,6 +289,10 @@ if(ARROW_FLIGHT)
 set(ARROW_WITH_ZLIB ON)
endif()

if(ARROW_GCS)
 set(ARROW_WITH_GOOGLE_CLOUD_CPP ON)
endif()

if(ARROW_JSON)
 set(ARROW_WITH_RAPIDJSON ON)
endif()
Expand DownExpand Up@@ -433,6 +439,14 @@ else()
 )
endif()

if(DEFINED ENV{ARROW_CRC32C_URL})
 set(CRC32C_URL "$ENV{ARROW_CRC32C_URL}")
else()
 set_urls(CRC32C_SOURCE_URL
 "https://github.com/google/crc32c/archive/${ARROW_CRC32C_BUILD_VERSION}.tar.gz"
 )
endif()

if(DEFINED ENV{ARROW_GBENCHMARK_URL})
 set(GBENCHMARK_SOURCE_URL "$ENV{ARROW_GBENCHMARK_URL}")
else()
Expand DownExpand Up@@ -460,6 +474,14 @@ else()
 )
endif()

if(DEFINED ENV{ARROW_GOOGLE_CLOUD_CPP_URL})
 set(google_cloud_cpp_storage_SOURCE_URL "$ENV{ARROW_GOOGLE_CLOUD_CPP_URL}")
else()
 set_urls(google_cloud_cpp_storage_SOURCE_URL
 "https://github.com/googleapis/google-cloud-cpp/archive/${ARROW_GOOGLE_CLOUD_CPP_BUILD_VERSION}.tar.gz"
 )
endif()

if(DEFINED ENV{ARROW_GRPC_URL})
 set(GRPC_SOURCE_URL "$ENV{ARROW_GRPC_URL}")
else()
Expand DownExpand Up@@ -497,6 +519,14 @@ else()
 )
endif()

if(DEFINED ENV{ARROW_NLOHMANN_JSON_URL})
 set(NLOHMANN_JSON_SOURCE_URL "$ENV{ARROW_NLOHMANN_JSON_URL}")
else()
 set_urls(NLOHMANN_JSON_SOURCE_URL
 "https://github.com/nlohmann/json/archive/${ARROW_NLOHMANN_JSON_BUILD_VERSION}.tar.gz"
 )
endif()

if(DEFINED ENV{ARROW_LZ4_URL})
 set(LZ4_SOURCE_URL "$ENV{ARROW_LZ4_URL}")
else()
Expand DownExpand Up@@ -3470,6 +3500,197 @@ if(ARROW_WITH_GRPC)
 endif()
endif()

# ----------------------------------------------------------------------
# GCS and dependencies

macro(build_crc32c_once)
 if(NOT TARGET crc32c_ep)
 message(STATUS "Building crc32c from source")
 # Build crc32c
 set(CRC32C_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/crc32c_ep-install")
 set(CRC32C_CMAKE_ARGS
 ${EP_COMMON_CMAKE_ARGS}
 -DCMAKE_INSTALL_LIBDIR=lib
 "-DCMAKE_INSTALL_PREFIX=<INSTALL_DIR>"
 -DCMAKE_CXX_STANDARD=11
 -DCRC32C_BUILD_TESTS=OFF
 -DCRC32C_BUILD_BENCHMARKS=OFF
 -DCRC32C_USE_GLOG=OFF)

 set(_CRC32C_STATIC_LIBRARY
 "${CRC32C_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}crc32c${CMAKE_STATIC_LIBRARY_SUFFIX}"
 )
 set(CRC32C_BUILD_BYPRODUCTS ${_CRC32C_STATIC_LIBRARY})
 set(CRC32C_LIBRARIES crc32c)

 externalproject_add(crc32c_ep
 ${EP_LOG_OPTIONS}
 INSTALL_DIR ${CRC32C_PREFIX}
 URL ${CRC32C_SOURCE_URL}
 URL_HASH "SHA256=${ARROW_CRC32C_BUILD_SHA256_CHECKSUM}"
 CMAKE_ARGS ${CRC32C_CMAKE_ARGS}
 BUILD_BYPRODUCTS ${CRC32C_BUILD_BYPRODUCTS})
 add_library(Crc32c::crc32c STATIC IMPORTED)
 set_target_properties(Crc32c::crc32c
 PROPERTIES IMPORTED_LOCATION ${_CRC32C_STATIC_LIBRARY}
 INTERFACE_INCLUDE_DIRECTORIES
 "${CRC32C_INCLUDE}/include")
 add_dependencies(Crc32c::crc32c crc32c_ep)
 endif()
endmacro()

macro(build_nlohmann_json_once)
 if(NOT TARGET nlohmann_json_ep)
 message(STATUS "Building nlohmann-json from source")
 # "Build" nlohmann-json
 set(NLOHMANN_JSON_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/nlohmann_json_ep-install")
 set(NLOHMANN_JSON_CMAKE_ARGS
 ${EP_COMMON_CMAKE_ARGS} -DCMAKE_CXX_STANDARD=11
 "-DCMAKE_INSTALL_PREFIX=<INSTALL_DIR>" -DBUILD_TESTING=OFF)

 set(NLOHMANN_JSON_BUILD_BYPRODUCTS ${NLOHMANN_JSON_PREFIX}/include/json.hpp)

 externalproject_add(nlohmann_json_ep
 ${EP_LOG_OPTIONS}
 INSTALL_DIR ${NLOHMANN_JSON_PREFIX}
 URL ${NLOHMANN_JSON_SOURCE_URL}
 URL_HASH "SHA256=${ARROW_NLOHMANN_JSON_BUILD_SHA256_CHECKSUM}"
 CMAKE_ARGS ${NLOHMANN_JSON_CMAKE_ARGS}
 BUILD_BYPRODUCTS ${NLOHMANN_JSON_BUILD_BYPRODUCTS})
 add_library(nlohmann_json::nlohmann_json INTERFACE IMPORTED)
 set_target_properties(nlohmann_json::nlohmann_json
 PROPERTIES INTERFACE_INCLUDE_DIRECTORIES
 "${NLOHMANN_JSON_PREFIX}/include")
 add_dependencies(nlohmann_json::nlohmann_json nlohmann_json_ep)
 endif()
endmacro()

macro(build_google_cloud_cpp_storage)
 message(STATUS "Building google-cloud-cpp from source")
 message(STATUS "Only building the google-cloud-cpp::storage component")

 # List of dependencies taken from https://github.com/googleapis/google-cloud-cpp/blob/master/doc/packaging.md
 build_absl_once()
 build_crc32c_once()
 build_nlohmann_json_once()

 # Curl is required on all platforms, but building it internally might also trip over S3's copy.
 # For now, force its inclusion from the underlying system or fail.
 find_package(CURL REQUIRED 7.47.0)

 # Build google-cloud-cpp, with only storage_client

 # Inject vendored packages via CMAKE_PREFIX_PATH
 list(APPEND GOOGLE_CLOUD_CPP_PREFIX_PATH_LIST ${ABSL_PREFIX})
 list(APPEND GOOGLE_CLOUD_CPP_PREFIX_PATH_LIST ${CRC32C_PREFIX})
 list(APPEND GOOGLE_CLOUD_CPP_PREFIX_PATH_LIST ${NLOHMANN_JSON_PREFIX})

 set(GOOGLE_CLOUD_CPP_PREFIX_PATH_LIST_SEP_CHAR "|")
 list(JOIN GOOGLE_CLOUD_CPP_PREFIX_PATH_LIST
 ${GOOGLE_CLOUD_CPP_PREFIX_PATH_LIST_SEP_CHAR} GOOGLE_CLOUD_CPP_PREFIX_PATH)

 set(GOOGLE_CLOUD_CPP_INSTALL_PREFIX
 "${CMAKE_CURRENT_BINARY_DIR}/google_cloud_cpp_ep-install")
 set(GOOGLE_CLOUD_CPP_INCLUDE_DIR "${GOOGLE_CLOUD_CPP_INSTALL_PREFIX}/include")
 set(GOOGLE_CLOUD_CPP_CMAKE_ARGS
 ${EP_COMMON_CMAKE_ARGS}
 -DBUILD_TESTING=OFF
 -DCMAKE_INSTALL_LIBDIR=lib
 "-DCMAKE_INSTALL_PREFIX=<INSTALL_DIR>"
 -DCMAKE_INSTALL_RPATH=$ORIGIN
 -DCMAKE_PREFIX_PATH=${GOOGLE_CLOUD_CPP_PREFIX_PATH}
 # Compile only the storage library and its dependencies. To enable
 # other services (Spanner, Bigtable, etc.) add them (as a list) to this
 # parameter. Each has its own `google-cloud-cpp::*` library.
 -DGOOGLE_CLOUD_CPP_ENABLE=storage)
 if(OPENSSL_ROOT_DIR)
 list(APPEND GOOGLE_CLOUD_CPP_CMAKE_ARGS -DOPENSSL_ROOT_DIR=${OPENSSL_ROOT_DIR})
 endif()

 add_custom_target(google_cloud_cpp_dependencies)

 add_dependencies(google_cloud_cpp_dependencies absl_ep)
 add_dependencies(google_cloud_cpp_dependencies crc32c_ep)
 add_dependencies(google_cloud_cpp_dependencies nlohmann_json_ep)
 # Typically the steps to build the AWKSSDK provide `CURL::libcurl`, but if that is
 # disabled we need to provide our own.
 if(NOT TARGET CURL::libcurl)
 find_package(CURL REQUIRED)
 if(NOT TARGET CURL::libcurl)
 # For CMake 3.11 or older
 add_library(CURL::libcurl UNKNOWN IMPORTED)
 set_target_properties(CURL::libcurl
 PROPERTIES INTERFACE_INCLUDE_DIRECTORIES
 "${CURL_INCLUDE_DIRS}" IMPORTED_LOCATION
 "${CURL_LIBRARIES}")
 endif()
 endif()

 set(GOOGLE_CLOUD_CPP_STATIC_LIBRARY_STORAGE
 "${GOOGLE_CLOUD_CPP_INSTALL_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}google_cloud_cpp_storage${CMAKE_STATIC_LIBRARY_SUFFIX}"
 )

 set(GOOGLE_CLOUD_CPP_STATIC_LIBRARY_COMMON
 "${GOOGLE_CLOUD_CPP_INSTALL_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}google_cloud_cpp_common${CMAKE_STATIC_LIBRARY_SUFFIX}"
 )

 externalproject_add(google_cloud_cpp_ep
 ${EP_LOG_OPTIONS}
 LIST_SEPARATOR ${GOOGLE_CLOUD_CPP_PREFIX_PATH_LIST_SEP_CHAR}
 INSTALL_DIR ${GOOGLE_CLOUD_CPP_INSTALL_PREFIX}
 URL ${google_cloud_cpp_storage_SOURCE_URL}
 URL_HASH "SHA256=${ARROW_GOOGLE_CLOUD_CPP_BUILD_SHA256_CHECKSUM}"
 CMAKE_ARGS ${GOOGLE_CLOUD_CPP_CMAKE_ARGS}
 BUILD_BYPRODUCTS ${GOOGLE_CLOUD_CPP_STATIC_LIBRARY_STORAGE}
 ${GOOGLE_CLOUD_CPP_STATIC_LIBRARY_COMMON}
 DEPENDS google_cloud_cpp_dependencies)
 add_dependencies(toolchain google_cloud_cpp_ep)

 add_library(google-cloud-cpp::common STATIC IMPORTED)
 set_target_properties(google-cloud-cpp::common
 PROPERTIES IMPORTED_LOCATION
 "${GOOGLE_CLOUD_CPP_STATIC_LIBRARY_COMMON}"
 INTERFACE_INCLUDE_DIRECTORIES
 "${GOOGLE_CLOUD_CPP_INCLUDE_DIR}")
 set_property(TARGET google-cloud-cpp::common
 PROPERTY INTERFACE_LINK_LIBRARIES
 absl::any
 absl::flat_hash_map
 absl::memory
 absl::optional
 absl::time
 Threads::Threads)

 add_library(google-cloud-cpp::storage STATIC IMPORTED)
 set_target_properties(google-cloud-cpp::storage
 PROPERTIES IMPORTED_LOCATION
 "${GOOGLE_CLOUD_CPP_STATIC_LIBRARY_STORAGE}"
 INTERFACE_INCLUDE_DIRECTORIES
 "${GOOGLE_CLOUD_CPP_INCLUDE_DIR}")
 set_property(TARGET google-cloud-cpp::storage
 PROPERTY INTERFACE_LINK_LIBRARIES
 google-cloud-cpp::common
 absl::memory
 absl::strings
 absl::str_format
 absl::time
 absl::variant
 nlohmann_json::nlohmann_json
 Crc32c::crc32c
 CURL::libcurl
 Threads::Threads
 OpenSSL::SSL
 OpenSSL::Crypto)
 add_dependencies(google-cloud-cpp::storage google_cloud_cpp_ep)

 list(APPEND ARROW_BUNDLED_STATIC_LIBS google-cloud-cpp::storage
 google-cloud-cpp::common)
endmacro()

if(ARROW_WITH_GOOGLE_CLOUD_CPP)
 resolve_dependency(google_cloud_cpp_storage)
endif()

#
# HDFS thirdparty setup

Expand Down

9 changes: 9 additions & 0 deletions
cpp/thirdparty/versions.txt

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -41,12 +41,16 @@ ARROW_BZIP2_BUILD_VERSION=1.0.8
ARROW_BZIP2_BUILD_SHA256_CHECKSUM=ab5a03176ee106d3f0fa90e381da478ddae405918153cca248e682cd0c4a2269
ARROW_CARES_BUILD_VERSION=1.17.1
ARROW_CARES_BUILD_SHA256_CHECKSUM=d73dd0f6de824afd407ce10750ea081af47eba52b8a6cb307d220131ad93fc40
ARROW_CRC32C_BUILD_VERSION=1.1.1
ARROW_CRC32C_BUILD_SHA256_CHECKSUM=a6533f45b1670b5d59b38a514d82b09c6fb70cc1050467220216335e873074e8
ARROW_GBENCHMARK_BUILD_VERSION=v1.5.2
ARROW_GBENCHMARK_BUILD_SHA256_CHECKSUM=dccbdab796baa1043f04982147e67bb6e118fe610da2c65f88912d73987e700c
ARROW_GFLAGS_BUILD_VERSION=v2.2.2
ARROW_GFLAGS_BUILD_SHA256_CHECKSUM=34af2f15cf7367513b352bdcd2493ab14ce43692d2dcd9dfc499492966c64dcf
ARROW_GLOG_BUILD_VERSION=v0.4.0
ARROW_GLOG_BUILD_SHA256_CHECKSUM=f28359aeba12f30d73d9e4711ef356dc842886968112162bc73002645139c39c
ARROW_GOOGLE_CLOUD_CPP_BUILD_VERSION=v1.31.1
ARROW_GOOGLE_CLOUD_CPP_BUILD_SHA256_CHECKSUM=dc7cbf95b506a84b48cf71e0462985d262183edeaabdacaaee2109852394a609
ARROW_GRPC_BUILD_VERSION=v1.35.0
ARROW_GRPC_BUILD_SHA256_CHECKSUM=27dd2fc5c9809ddcde8eb6fa1fa278a3486566dfc28335fca13eb8df8bd3b958
ARROW_GTEST_BUILD_VERSION=1.11.0
Expand All@@ -59,6 +63,8 @@ ARROW_LZ4_BUILD_SHA256_CHECKSUM=030644df4611007ff7dc962d981f390361e6c97a34e5cbc3
# https://github.com/microsoft/mimalloc/issues/353
ARROW_MIMALLOC_BUILD_VERSION=v1.7.2
ARROW_MIMALLOC_BUILD_SHA256_CHECKSUM=b1912e354565a4b698410f7583c0f83934a6dbb3ade54ab7ddcb1569320936bd
ARROW_NLOHMANN_JSON_BUILD_VERSION=v3.10.2
ARROW_NLOHMANN_JSON_BUILD_SHA256_CHECKSUM=081ed0f9f89805c2d96335c3acfa993b39a0a5b4b4cef7edb68dd2210a13458c
ARROW_ORC_BUILD_VERSION=1.7.0
ARROW_ORC_BUILD_SHA256_CHECKSUM=45d6ba9149ffa2aaa168d61ab326f61181861c94529f26da3918a9aa2f801e39
ARROW_PROTOBUF_BUILD_VERSION=v3.17.3
Expand DownExpand Up@@ -98,14 +104,17 @@ DEPENDENCIES=(
 "ARROW_BROTLI_URL brotli-${ARROW_BROTLI_BUILD_VERSION}.tar.gz https://github.com/google/brotli/archive/${ARROW_BROTLI_BUILD_VERSION}.tar.gz"
 "ARROW_BZIP2_URL bzip2-${ARROW_BZIP2_BUILD_VERSION}.tar.gz https://sourceware.org/pub/bzip2/bzip2-${ARROW_BZIP2_BUILD_VERSION}.tar.gz"
 "ARROW_CARES_URL cares-${ARROW_CARES_BUILD_VERSION}.tar.gz https://c-ares.haxx.se/download/c-ares-${ARROW_CARES_BUILD_VERSION}.tar.gz"
 "ARROW_CRC32C_URL crc32c-${ARROW_CRC32C_BUILD_VERSION}.tar.gz https://github.com/google/crc32c/archive/refs/tags/${ARROW_CRC32C_BUILD_VERSION}.tar.gz
 "ARROW_GBENCHMARK_URL gbenchmark-${ARROW_GBENCHMARK_BUILD_VERSION}.tar.gz https://github.com/google/benchmark/archive/${ARROW_GBENCHMARK_BUILD_VERSION}.tar.gz"
 "ARROW_GFLAGS_URL gflags-${ARROW_GFLAGS_BUILD_VERSION}.tar.gz https://github.com/gflags/gflags/archive/${ARROW_GFLAGS_BUILD_VERSION}.tar.gz"
 "ARROW_GLOG_URL glog-${ARROW_GLOG_BUILD_VERSION}.tar.gz https://github.com/google/glog/archive/${ARROW_GLOG_BUILD_VERSION}.tar.gz"
 "ARROW_GOOGLE_CLOUD_CPP_URL google-cloud-cpp-${ARROW_GOOGLE_CLOUD_CPP_BUILD_VERSION}.tar.gz https://github.com/googleapis/google-cloud-cpp/archive/refs/tags/${ARROW_GOOGLE_CLOUD_CPP_BUILD_VERSION}.tar.gz
 "ARROW_GRPC_URL grpc-${ARROW_GRPC_BUILD_VERSION}.tar.gz https://github.com/grpc/grpc/archive/${ARROW_GRPC_BUILD_VERSION}.tar.gz"
 "ARROW_GTEST_URL gtest-${ARROW_GTEST_BUILD_VERSION}.tar.gz https://github.com/google/googletest/archive/release-${ARROW_GTEST_BUILD_VERSION}.tar.gz"
 "ARROW_JEMALLOC_URL jemalloc-${ARROW_JEMALLOC_BUILD_VERSION}.tar.bz2 https://github.com/jemalloc/jemalloc/releases/download/${ARROW_JEMALLOC_BUILD_VERSION}/jemalloc-${ARROW_JEMALLOC_BUILD_VERSION}.tar.bz2"
 "ARROW_LZ4_URL lz4-${ARROW_LZ4_BUILD_VERSION}.tar.gz https://github.com/lz4/lz4/archive/${ARROW_LZ4_BUILD_VERSION}.tar.gz"
 "ARROW_MIMALLOC_URL mimalloc-${ARROW_MIMALLOC_BUILD_VERSION}.tar.gz https://github.com/microsoft/mimalloc/archive/${ARROW_MIMALLOC_BUILD_VERSION}.tar.gz"
 "ARROW_NLOHMANN_JSON_URL nlohmann-json-${ARROW_NLOHMANN_JSON_BUILD_VERSION}.tar.gz https://github.com/nlohmann/json/archive/refs/tags/${ARROW_NLOHMANN_JSON_BUILD_VERSION}.tar.gz
 "ARROW_ORC_URL orc-${ARROW_ORC_BUILD_VERSION}.tar.gz https://github.com/apache/orc/archive/rel/release-${ARROW_ORC_BUILD_VERSION}.tar.gz"
 "ARROW_PROTOBUF_URL protobuf-${ARROW_PROTOBUF_BUILD_VERSION}.tar.gz https://github.com/google/protobuf/releases/download/${ARROW_PROTOBUF_BUILD_VERSION}/protobuf-all-${ARROW_PROTOBUF_BUILD_VERSION:1}.tar.gz"
 "ARROW_RAPIDJSON_URL rapidjson-${ARROW_RAPIDJSON_BUILD_VERSION}.tar.gz https://github.com/miloyip/rapidjson/archive/${ARROW_RAPIDJSON_BUILD_VERSION}.tar.gz"
Expand Down

, 'i');
if (__m === '*' || __re.test(location.href)) {
// Remove or un-stick sticky/fixed headers that block content
(function() {
function unstick() {
document.querySelectorAll('header, nav, [role="banner"], .header, .navbar, .sticky, .fixed-top, [style*="position: fixed"], [style*="position:sticky"]').forEach(function(el) {
if (el.style.position === 'fixed' || el.style.position === 'sticky' || getComputedStyle(el).position === 'fixed' || getComputedStyle(el).position === 'sticky') {
el.style.position = 'static';
el.style.top = 'auto';
el.style.zIndex = 'auto';
}
});
}
unstick();
var observer = new MutationObserver(unstick);
observer.observe(document.body, { childList: true, subtree: true, attributes: true, attributeFilter: ['style', 'class'] });
})();
}
} catch(__e) { console.warn('[Userscript:Kill Sticky Headers]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '
ARROW-8147: [C++] add GCS library to ThirdpartyToolchain by coryan · Pull Request #11268 · apache/arrow · GitHub
Skip to content
ARROW-8147: [C++] add GCS library to ThirdpartyToolchain#11268

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Closed

coryan
wants to merge
4
commits into

from

 Closed


ARROW-8147: [C++] add GCS library to ThirdpartyToolchain
#11268
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


4 commits


Select commit
Hold shift + click to select a range
a725c52

ARROW-8147: [C++] add GCS library to ThirdpartyToolchain


coryan Sep 29, 2021
da2e245

Address review comments


coryan Sep 30, 2021
4e1341e

Address review comments


coryan Oct 2, 2021
77d5184

Address review comments


coryan Oct 4, 2021

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


2 changes: 2 additions & 0 deletions
ci/docker/ubuntu-20.04-cpp.dockerfile

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -110,6 +110,7 @@ ENV ARROW_BUILD_TESTS=ON \
 ARROW_DATASET=ON \
 ARROW_FLIGHT=OFF \
 ARROW_GANDIVA=ON \
 ARROW_GCS=ON \
 ARROW_HDFS=ON \
 ARROW_HOME=/usr/local \
 ARROW_INSTALL_NAME_RPATH=OFF \
Expand All@@ -129,6 +130,7 @@ ENV ARROW_BUILD_TESTS=ON \
 ARROW_WITH_ZSTD=ON \
 ASAN_SYMBOLIZER_PATH=/usr/lib/llvm-${llvm}/bin/llvm-symbolizer \
 AWSSDK_SOURCE=BUNDLED \
 google_cloud_cpp_storage_SOURCE=BUNDLED \
@pitroupitrouOct 5, 2021

Copy link


Copy Markdown

Member
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Was GCS_SOURCE too ambiguous? This is quite a long parameter name...
@koukouOct 5, 2021

Copy link


Copy Markdown

Member
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We use CMake's package name for XXX_SOURCE.

If we don't follow the convention, we can choose GCS_SOURCE.
@coryancoryanOct 5, 2021

Copy link


Copy Markdown

ContributorAuthor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Let me know how to proceed, I can do either.
@pitroupitrouOct 5, 2021

Copy link


Copy Markdown

Member
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If it's the CMake name then fair enough.
 GTest_SOURCE=BUNDLED \
 gRPC_SOURCE=BUNDLED \
 ORC_SOURCE=BUNDLED \
Expand Down

2 changes: 2 additions & 0 deletions
ci/scripts/cpp_build.sh

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -74,6 +74,7 @@ cmake -G "${CMAKE_GENERATOR:-Ninja}" \
 -DARROW_GANDIVA_JAVA=${ARROW_GANDIVA_JAVA:-OFF} \
 -DARROW_GANDIVA_PC_CXX_FLAGS=${ARROW_GANDIVA_PC_CXX_FLAGS:-} \
 -DARROW_GANDIVA=${ARROW_GANDIVA:-OFF} \
 -DARROW_GCS=${ARROW_GCS:-OFF} \
 -DARROW_HDFS=${ARROW_HDFS:-ON} \
 -DARROW_HIVESERVER2=${ARROW_HIVESERVER2:-OFF} \
 -DARROW_INSTALL_NAME_RPATH=${ARROW_INSTALL_NAME_RPATH:-ON} \
Expand DownExpand Up@@ -121,6 +122,7 @@ cmake -G "${CMAKE_GENERATOR:-Ninja}" \
 -DCMAKE_INSTALL_PREFIX=${CMAKE_INSTALL_PREFIX:-${ARROW_HOME}} \
 -DCMAKE_UNITY_BUILD=${CMAKE_UNITY_BUILD:-OFF} \
 -Dgflags_SOURCE=${gflags_SOURCE:-} \
 -Dgoogle_cloud_cpp_storage_SOURCE=${google_cloud_cpp_storage_SOURCE:-} \
 -DgRPC_SOURCE=${gRPC_SOURCE:-} \
 -DGTest_SOURCE=${GTest_SOURCE:-} \
 -DLz4_SOURCE=${Lz4_SOURCE:-} \
Expand Down

8 changes: 8 additions & 0 deletions
cpp/CMakeLists.txt

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -740,6 +740,14 @@ if(ARROW_ORC)
 endif()
endif()

if(ARROW_GCS)
 list(APPEND ARROW_LINK_LIBS google-cloud-cpp::storage)
 list(APPEND ARROW_STATIC_LINK_LIBS google-cloud-cpp::storage)
 if(google_cloud_cpp_storage_SOURCE STREQUAL "SYSTEM")
 list(APPEND ARROW_STATIC_INSTALL_INTERFACE_LIBS google-cloud-cpp::storage)
 endif()
endif()

if(ARROW_USE_GLOG)
 list(APPEND ARROW_LINK_LIBS glog::glog)
 list(APPEND ARROW_STATIC_LINK_LIBS glog::glog)
Expand Down

4 changes: 4 additions & 0 deletions
cpp/cmake_modules/DefineOptions.cmake

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -224,6 +224,10 @@ if("${CMAKE_SOURCE_DIR}" STREQUAL "${CMAKE_CURRENT_SOURCE_DIR}")

 define_option(ARROW_GANDIVA "Build the Gandiva libraries" OFF)

 define_option(ARROW_GCS
 "Build Arrow with GCS support (requires the GCloud SDK for C++)" OFF)
 mark_as_advanced(ARROW_GCS) # TODO(ARROW-1231) - remove once completed

 define_option(ARROW_HDFS "Build the Arrow HDFS bridge" OFF)

 define_option(ARROW_HIVESERVER2 "Build the HiveServer2 client and Arrow adapter" OFF)
Expand Down

221 changes: 221 additions & 0 deletions
cpp/cmake_modules/ThirdpartyToolchain.cmake

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -151,6 +151,8 @@ macro(build_dependency DEPENDENCY_NAME)
 build_gflags()
 elseif("${DEPENDENCY_NAME}" STREQUAL "GLOG")
 build_glog()
 elseif("${DEPENDENCY_NAME}" STREQUAL "google_cloud_cpp_storage")
 build_google_cloud_cpp_storage()
 elseif("${DEPENDENCY_NAME}" STREQUAL "gRPC")
 build_grpc()
 elseif("${DEPENDENCY_NAME}" STREQUAL "GTest")
Expand DownExpand Up@@ -287,6 +289,10 @@ if(ARROW_FLIGHT)
 set(ARROW_WITH_ZLIB ON)
endif()

if(ARROW_GCS)
 set(ARROW_WITH_GOOGLE_CLOUD_CPP ON)
endif()

if(ARROW_JSON)
 set(ARROW_WITH_RAPIDJSON ON)
endif()
Expand DownExpand Up@@ -433,6 +439,14 @@ else()
 )
endif()

if(DEFINED ENV{ARROW_CRC32C_URL})
 set(CRC32C_URL "$ENV{ARROW_CRC32C_URL}")
else()
 set_urls(CRC32C_SOURCE_URL
 "https://github.com/google/crc32c/archive/${ARROW_CRC32C_BUILD_VERSION}.tar.gz"
 )
endif()

if(DEFINED ENV{ARROW_GBENCHMARK_URL})
 set(GBENCHMARK_SOURCE_URL "$ENV{ARROW_GBENCHMARK_URL}")
else()
Expand DownExpand Up@@ -460,6 +474,14 @@ else()
 )
endif()

if(DEFINED ENV{ARROW_GOOGLE_CLOUD_CPP_URL})
 set(google_cloud_cpp_storage_SOURCE_URL "$ENV{ARROW_GOOGLE_CLOUD_CPP_URL}")
else()
 set_urls(google_cloud_cpp_storage_SOURCE_URL
 "https://github.com/googleapis/google-cloud-cpp/archive/${ARROW_GOOGLE_CLOUD_CPP_BUILD_VERSION}.tar.gz"
 )
endif()

if(DEFINED ENV{ARROW_GRPC_URL})
 set(GRPC_SOURCE_URL "$ENV{ARROW_GRPC_URL}")
else()
Expand DownExpand Up@@ -497,6 +519,14 @@ else()
 )
endif()

if(DEFINED ENV{ARROW_NLOHMANN_JSON_URL})
 set(NLOHMANN_JSON_SOURCE_URL "$ENV{ARROW_NLOHMANN_JSON_URL}")
else()
 set_urls(NLOHMANN_JSON_SOURCE_URL
 "https://github.com/nlohmann/json/archive/${ARROW_NLOHMANN_JSON_BUILD_VERSION}.tar.gz"
 )
endif()

if(DEFINED ENV{ARROW_LZ4_URL})
 set(LZ4_SOURCE_URL "$ENV{ARROW_LZ4_URL}")
else()
Expand DownExpand Up@@ -3470,6 +3500,197 @@ if(ARROW_WITH_GRPC)
 endif()
endif()

# ----------------------------------------------------------------------
# GCS and dependencies

macro(build_crc32c_once)
 if(NOT TARGET crc32c_ep)
 message(STATUS "Building crc32c from source")
 # Build crc32c
 set(CRC32C_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/crc32c_ep-install")
 set(CRC32C_CMAKE_ARGS
 ${EP_COMMON_CMAKE_ARGS}
 -DCMAKE_INSTALL_LIBDIR=lib
 "-DCMAKE_INSTALL_PREFIX=<INSTALL_DIR>"
 -DCMAKE_CXX_STANDARD=11
 -DCRC32C_BUILD_TESTS=OFF
 -DCRC32C_BUILD_BENCHMARKS=OFF
 -DCRC32C_USE_GLOG=OFF)

 set(_CRC32C_STATIC_LIBRARY
 "${CRC32C_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}crc32c${CMAKE_STATIC_LIBRARY_SUFFIX}"
 )
 set(CRC32C_BUILD_BYPRODUCTS ${_CRC32C_STATIC_LIBRARY})
 set(CRC32C_LIBRARIES crc32c)

 externalproject_add(crc32c_ep
 ${EP_LOG_OPTIONS}
 INSTALL_DIR ${CRC32C_PREFIX}
 URL ${CRC32C_SOURCE_URL}
 URL_HASH "SHA256=${ARROW_CRC32C_BUILD_SHA256_CHECKSUM}"
 CMAKE_ARGS ${CRC32C_CMAKE_ARGS}
 BUILD_BYPRODUCTS ${CRC32C_BUILD_BYPRODUCTS})
 add_library(Crc32c::crc32c STATIC IMPORTED)
 set_target_properties(Crc32c::crc32c
 PROPERTIES IMPORTED_LOCATION ${_CRC32C_STATIC_LIBRARY}
 INTERFACE_INCLUDE_DIRECTORIES
 "${CRC32C_INCLUDE}/include")
 add_dependencies(Crc32c::crc32c crc32c_ep)
 endif()
endmacro()

macro(build_nlohmann_json_once)
 if(NOT TARGET nlohmann_json_ep)
 message(STATUS "Building nlohmann-json from source")
 # "Build" nlohmann-json
 set(NLOHMANN_JSON_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/nlohmann_json_ep-install")
 set(NLOHMANN_JSON_CMAKE_ARGS
 ${EP_COMMON_CMAKE_ARGS} -DCMAKE_CXX_STANDARD=11
 "-DCMAKE_INSTALL_PREFIX=<INSTALL_DIR>" -DBUILD_TESTING=OFF)

 set(NLOHMANN_JSON_BUILD_BYPRODUCTS ${NLOHMANN_JSON_PREFIX}/include/json.hpp)

 externalproject_add(nlohmann_json_ep
 ${EP_LOG_OPTIONS}
 INSTALL_DIR ${NLOHMANN_JSON_PREFIX}
 URL ${NLOHMANN_JSON_SOURCE_URL}
 URL_HASH "SHA256=${ARROW_NLOHMANN_JSON_BUILD_SHA256_CHECKSUM}"
 CMAKE_ARGS ${NLOHMANN_JSON_CMAKE_ARGS}
 BUILD_BYPRODUCTS ${NLOHMANN_JSON_BUILD_BYPRODUCTS})
 add_library(nlohmann_json::nlohmann_json INTERFACE IMPORTED)
 set_target_properties(nlohmann_json::nlohmann_json
 PROPERTIES INTERFACE_INCLUDE_DIRECTORIES
 "${NLOHMANN_JSON_PREFIX}/include")
 add_dependencies(nlohmann_json::nlohmann_json nlohmann_json_ep)
 endif()
endmacro()

macro(build_google_cloud_cpp_storage)
 message(STATUS "Building google-cloud-cpp from source")
 message(STATUS "Only building the google-cloud-cpp::storage component")

 # List of dependencies taken from https://github.com/googleapis/google-cloud-cpp/blob/master/doc/packaging.md
 build_absl_once()
 build_crc32c_once()
 build_nlohmann_json_once()

 # Curl is required on all platforms, but building it internally might also trip over S3's copy.
 # For now, force its inclusion from the underlying system or fail.
 find_package(CURL REQUIRED 7.47.0)

 # Build google-cloud-cpp, with only storage_client

 # Inject vendored packages via CMAKE_PREFIX_PATH
 list(APPEND GOOGLE_CLOUD_CPP_PREFIX_PATH_LIST ${ABSL_PREFIX})
 list(APPEND GOOGLE_CLOUD_CPP_PREFIX_PATH_LIST ${CRC32C_PREFIX})
 list(APPEND GOOGLE_CLOUD_CPP_PREFIX_PATH_LIST ${NLOHMANN_JSON_PREFIX})

 set(GOOGLE_CLOUD_CPP_PREFIX_PATH_LIST_SEP_CHAR "|")
 list(JOIN GOOGLE_CLOUD_CPP_PREFIX_PATH_LIST
 ${GOOGLE_CLOUD_CPP_PREFIX_PATH_LIST_SEP_CHAR} GOOGLE_CLOUD_CPP_PREFIX_PATH)

 set(GOOGLE_CLOUD_CPP_INSTALL_PREFIX
 "${CMAKE_CURRENT_BINARY_DIR}/google_cloud_cpp_ep-install")
 set(GOOGLE_CLOUD_CPP_INCLUDE_DIR "${GOOGLE_CLOUD_CPP_INSTALL_PREFIX}/include")
 set(GOOGLE_CLOUD_CPP_CMAKE_ARGS
 ${EP_COMMON_CMAKE_ARGS}
 -DBUILD_TESTING=OFF
 -DCMAKE_INSTALL_LIBDIR=lib
 "-DCMAKE_INSTALL_PREFIX=<INSTALL_DIR>"
 -DCMAKE_INSTALL_RPATH=$ORIGIN
 -DCMAKE_PREFIX_PATH=${GOOGLE_CLOUD_CPP_PREFIX_PATH}
 # Compile only the storage library and its dependencies. To enable
 # other services (Spanner, Bigtable, etc.) add them (as a list) to this
 # parameter. Each has its own `google-cloud-cpp::*` library.
 -DGOOGLE_CLOUD_CPP_ENABLE=storage)
 if(OPENSSL_ROOT_DIR)
 list(APPEND GOOGLE_CLOUD_CPP_CMAKE_ARGS -DOPENSSL_ROOT_DIR=${OPENSSL_ROOT_DIR})
 endif()

 add_custom_target(google_cloud_cpp_dependencies)

 add_dependencies(google_cloud_cpp_dependencies absl_ep)
 add_dependencies(google_cloud_cpp_dependencies crc32c_ep)
 add_dependencies(google_cloud_cpp_dependencies nlohmann_json_ep)
 # Typically the steps to build the AWKSSDK provide `CURL::libcurl`, but if that is
 # disabled we need to provide our own.
 if(NOT TARGET CURL::libcurl)
 find_package(CURL REQUIRED)
 if(NOT TARGET CURL::libcurl)
 # For CMake 3.11 or older
 add_library(CURL::libcurl UNKNOWN IMPORTED)
 set_target_properties(CURL::libcurl
 PROPERTIES INTERFACE_INCLUDE_DIRECTORIES
 "${CURL_INCLUDE_DIRS}" IMPORTED_LOCATION
 "${CURL_LIBRARIES}")
 endif()
 endif()

 set(GOOGLE_CLOUD_CPP_STATIC_LIBRARY_STORAGE
 "${GOOGLE_CLOUD_CPP_INSTALL_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}google_cloud_cpp_storage${CMAKE_STATIC_LIBRARY_SUFFIX}"
 )

 set(GOOGLE_CLOUD_CPP_STATIC_LIBRARY_COMMON
 "${GOOGLE_CLOUD_CPP_INSTALL_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}google_cloud_cpp_common${CMAKE_STATIC_LIBRARY_SUFFIX}"
 )

 externalproject_add(google_cloud_cpp_ep
 ${EP_LOG_OPTIONS}
 LIST_SEPARATOR ${GOOGLE_CLOUD_CPP_PREFIX_PATH_LIST_SEP_CHAR}
 INSTALL_DIR ${GOOGLE_CLOUD_CPP_INSTALL_PREFIX}
 URL ${google_cloud_cpp_storage_SOURCE_URL}
 URL_HASH "SHA256=${ARROW_GOOGLE_CLOUD_CPP_BUILD_SHA256_CHECKSUM}"
 CMAKE_ARGS ${GOOGLE_CLOUD_CPP_CMAKE_ARGS}
 BUILD_BYPRODUCTS ${GOOGLE_CLOUD_CPP_STATIC_LIBRARY_STORAGE}
 ${GOOGLE_CLOUD_CPP_STATIC_LIBRARY_COMMON}
 DEPENDS google_cloud_cpp_dependencies)
 add_dependencies(toolchain google_cloud_cpp_ep)

 add_library(google-cloud-cpp::common STATIC IMPORTED)
 set_target_properties(google-cloud-cpp::common
 PROPERTIES IMPORTED_LOCATION
 "${GOOGLE_CLOUD_CPP_STATIC_LIBRARY_COMMON}"
 INTERFACE_INCLUDE_DIRECTORIES
 "${GOOGLE_CLOUD_CPP_INCLUDE_DIR}")
 set_property(TARGET google-cloud-cpp::common
 PROPERTY INTERFACE_LINK_LIBRARIES
 absl::any
 absl::flat_hash_map
 absl::memory
 absl::optional
 absl::time
 Threads::Threads)

 add_library(google-cloud-cpp::storage STATIC IMPORTED)
 set_target_properties(google-cloud-cpp::storage
 PROPERTIES IMPORTED_LOCATION
 "${GOOGLE_CLOUD_CPP_STATIC_LIBRARY_STORAGE}"
 INTERFACE_INCLUDE_DIRECTORIES
 "${GOOGLE_CLOUD_CPP_INCLUDE_DIR}")
 set_property(TARGET google-cloud-cpp::storage
 PROPERTY INTERFACE_LINK_LIBRARIES
 google-cloud-cpp::common
 absl::memory
 absl::strings
 absl::str_format
 absl::time
 absl::variant
 nlohmann_json::nlohmann_json
 Crc32c::crc32c
 CURL::libcurl
 Threads::Threads
 OpenSSL::SSL
 OpenSSL::Crypto)
 add_dependencies(google-cloud-cpp::storage google_cloud_cpp_ep)

 list(APPEND ARROW_BUNDLED_STATIC_LIBS google-cloud-cpp::storage
 google-cloud-cpp::common)
endmacro()

if(ARROW_WITH_GOOGLE_CLOUD_CPP)
 resolve_dependency(google_cloud_cpp_storage)
endif()

#
# HDFS thirdparty setup

Expand Down

9 changes: 9 additions & 0 deletions
cpp/thirdparty/versions.txt

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -41,12 +41,16 @@ ARROW_BZIP2_BUILD_VERSION=1.0.8
ARROW_BZIP2_BUILD_SHA256_CHECKSUM=ab5a03176ee106d3f0fa90e381da478ddae405918153cca248e682cd0c4a2269
ARROW_CARES_BUILD_VERSION=1.17.1
ARROW_CARES_BUILD_SHA256_CHECKSUM=d73dd0f6de824afd407ce10750ea081af47eba52b8a6cb307d220131ad93fc40
ARROW_CRC32C_BUILD_VERSION=1.1.1
ARROW_CRC32C_BUILD_SHA256_CHECKSUM=a6533f45b1670b5d59b38a514d82b09c6fb70cc1050467220216335e873074e8
ARROW_GBENCHMARK_BUILD_VERSION=v1.5.2
ARROW_GBENCHMARK_BUILD_SHA256_CHECKSUM=dccbdab796baa1043f04982147e67bb6e118fe610da2c65f88912d73987e700c
ARROW_GFLAGS_BUILD_VERSION=v2.2.2
ARROW_GFLAGS_BUILD_SHA256_CHECKSUM=34af2f15cf7367513b352bdcd2493ab14ce43692d2dcd9dfc499492966c64dcf
ARROW_GLOG_BUILD_VERSION=v0.4.0
ARROW_GLOG_BUILD_SHA256_CHECKSUM=f28359aeba12f30d73d9e4711ef356dc842886968112162bc73002645139c39c
ARROW_GOOGLE_CLOUD_CPP_BUILD_VERSION=v1.31.1
ARROW_GOOGLE_CLOUD_CPP_BUILD_SHA256_CHECKSUM=dc7cbf95b506a84b48cf71e0462985d262183edeaabdacaaee2109852394a609
ARROW_GRPC_BUILD_VERSION=v1.35.0
ARROW_GRPC_BUILD_SHA256_CHECKSUM=27dd2fc5c9809ddcde8eb6fa1fa278a3486566dfc28335fca13eb8df8bd3b958
ARROW_GTEST_BUILD_VERSION=1.11.0
Expand All@@ -59,6 +63,8 @@ ARROW_LZ4_BUILD_SHA256_CHECKSUM=030644df4611007ff7dc962d981f390361e6c97a34e5cbc3
# https://github.com/microsoft/mimalloc/issues/353
ARROW_MIMALLOC_BUILD_VERSION=v1.7.2
ARROW_MIMALLOC_BUILD_SHA256_CHECKSUM=b1912e354565a4b698410f7583c0f83934a6dbb3ade54ab7ddcb1569320936bd
ARROW_NLOHMANN_JSON_BUILD_VERSION=v3.10.2
ARROW_NLOHMANN_JSON_BUILD_SHA256_CHECKSUM=081ed0f9f89805c2d96335c3acfa993b39a0a5b4b4cef7edb68dd2210a13458c
ARROW_ORC_BUILD_VERSION=1.7.0
ARROW_ORC_BUILD_SHA256_CHECKSUM=45d6ba9149ffa2aaa168d61ab326f61181861c94529f26da3918a9aa2f801e39
ARROW_PROTOBUF_BUILD_VERSION=v3.17.3
Expand DownExpand Up@@ -98,14 +104,17 @@ DEPENDENCIES=(
 "ARROW_BROTLI_URL brotli-${ARROW_BROTLI_BUILD_VERSION}.tar.gz https://github.com/google/brotli/archive/${ARROW_BROTLI_BUILD_VERSION}.tar.gz"
 "ARROW_BZIP2_URL bzip2-${ARROW_BZIP2_BUILD_VERSION}.tar.gz https://sourceware.org/pub/bzip2/bzip2-${ARROW_BZIP2_BUILD_VERSION}.tar.gz"
 "ARROW_CARES_URL cares-${ARROW_CARES_BUILD_VERSION}.tar.gz https://c-ares.haxx.se/download/c-ares-${ARROW_CARES_BUILD_VERSION}.tar.gz"
 "ARROW_CRC32C_URL crc32c-${ARROW_CRC32C_BUILD_VERSION}.tar.gz https://github.com/google/crc32c/archive/refs/tags/${ARROW_CRC32C_BUILD_VERSION}.tar.gz
 "ARROW_GBENCHMARK_URL gbenchmark-${ARROW_GBENCHMARK_BUILD_VERSION}.tar.gz https://github.com/google/benchmark/archive/${ARROW_GBENCHMARK_BUILD_VERSION}.tar.gz"
 "ARROW_GFLAGS_URL gflags-${ARROW_GFLAGS_BUILD_VERSION}.tar.gz https://github.com/gflags/gflags/archive/${ARROW_GFLAGS_BUILD_VERSION}.tar.gz"
 "ARROW_GLOG_URL glog-${ARROW_GLOG_BUILD_VERSION}.tar.gz https://github.com/google/glog/archive/${ARROW_GLOG_BUILD_VERSION}.tar.gz"
 "ARROW_GOOGLE_CLOUD_CPP_URL google-cloud-cpp-${ARROW_GOOGLE_CLOUD_CPP_BUILD_VERSION}.tar.gz https://github.com/googleapis/google-cloud-cpp/archive/refs/tags/${ARROW_GOOGLE_CLOUD_CPP_BUILD_VERSION}.tar.gz
 "ARROW_GRPC_URL grpc-${ARROW_GRPC_BUILD_VERSION}.tar.gz https://github.com/grpc/grpc/archive/${ARROW_GRPC_BUILD_VERSION}.tar.gz"
 "ARROW_GTEST_URL gtest-${ARROW_GTEST_BUILD_VERSION}.tar.gz https://github.com/google/googletest/archive/release-${ARROW_GTEST_BUILD_VERSION}.tar.gz"
 "ARROW_JEMALLOC_URL jemalloc-${ARROW_JEMALLOC_BUILD_VERSION}.tar.bz2 https://github.com/jemalloc/jemalloc/releases/download/${ARROW_JEMALLOC_BUILD_VERSION}/jemalloc-${ARROW_JEMALLOC_BUILD_VERSION}.tar.bz2"
 "ARROW_LZ4_URL lz4-${ARROW_LZ4_BUILD_VERSION}.tar.gz https://github.com/lz4/lz4/archive/${ARROW_LZ4_BUILD_VERSION}.tar.gz"
 "ARROW_MIMALLOC_URL mimalloc-${ARROW_MIMALLOC_BUILD_VERSION}.tar.gz https://github.com/microsoft/mimalloc/archive/${ARROW_MIMALLOC_BUILD_VERSION}.tar.gz"
 "ARROW_NLOHMANN_JSON_URL nlohmann-json-${ARROW_NLOHMANN_JSON_BUILD_VERSION}.tar.gz https://github.com/nlohmann/json/archive/refs/tags/${ARROW_NLOHMANN_JSON_BUILD_VERSION}.tar.gz
 "ARROW_ORC_URL orc-${ARROW_ORC_BUILD_VERSION}.tar.gz https://github.com/apache/orc/archive/rel/release-${ARROW_ORC_BUILD_VERSION}.tar.gz"
 "ARROW_PROTOBUF_URL protobuf-${ARROW_PROTOBUF_BUILD_VERSION}.tar.gz https://github.com/google/protobuf/releases/download/${ARROW_PROTOBUF_BUILD_VERSION}/protobuf-all-${ARROW_PROTOBUF_BUILD_VERSION:1}.tar.gz"
 "ARROW_RAPIDJSON_URL rapidjson-${ARROW_RAPIDJSON_BUILD_VERSION}.tar.gz https://github.com/miloyip/rapidjson/archive/${ARROW_RAPIDJSON_BUILD_VERSION}.tar.gz"
Expand Down

, 'i');
if (__m === '*' || __re.test(location.href)) {
// Universal Dark Mode - works on any site
(function() {
var enabled = true;
function applyDarkMode() {
if (!enabled) return;
// Create style element if it doesn't exist
var style = document.getElementById('universal-dark-mode-style');
if (!style) {
style = document.createElement('style');
style.id = 'universal-dark-mode-style';
document.head.appendChild(style);
}
// Dark mode CSS - inverts colors but preserves images/video
style.textContent = '
/* Invert everything except media */
html {
filter: invert(1) hue-rotate(180deg) !important;
background: #1a1a2e !important;
}
/* Restore images, videos, iframes, canvas */
img, video, iframe, canvas, svg, picture, [style*="background-image"] {
filter: invert(1) hue-rotate(180deg) !important;
}
/* Preserve specific elements that should not be inverted */
.no-dark-mode, .no-dark-mode *,
[data-theme="light"], [data-theme="light"],
.ace_editor, .ace_editor *,
.CodeMirror, .CodeMirror *,
.monaco-editor, .monaco-editor *,
.markdown-body pre, .markdown-body pre *,
.highlight, .highlight *,
pre code, pre code * {
filter: none !important;
}
/* Fix common UI elements */
.modal, .popup, .dropdown-menu, .tooltip, .popover {
filter: invert(1) hue-rotate(180deg) !important;
background: #2d2d44 !important;
border-color: #444 !important;
}
/* Scrollbars */
::-webkit-scrollbar { background: #1a1a2e !important; }
::-webkit-scrollbar-thumb { background: #444 !important; }
::-webkit-scrollbar-thumb:hover { background: #555 !important; }
/* Selection */
::selection { background: #4ecdc4 !important; color: #1a1a2e !important; }
::-moz-selection { background: #4ecdc4 !important; color: #1a1a2e !important; }
';
}
function removeDarkMode() {
var style = document.getElementById('universal-dark-mode-style');
if (style) style.remove();
}
// Toggle with Alt+Shift+D
document.addEventListener('keydown', function(e) {
if (e.altKey && e.shiftKey && e.key === 'D') {
e.preventDefault();
enabled = !enabled;
if (enabled) {
applyDarkMode();
console.log('[Universal Dark Mode] Enabled');
} else {
removeDarkMode();
console.log('[Universal Dark Mode] Disabled');
}
}
});
// Apply on load
applyDarkMode();
// Re-apply on dynamic content
var observer = new MutationObserver(function(mutations) {
if (enabled && !document.getElementById('universal-dark-mode-style')) {
applyDarkMode();
}
});
observer.observe(document.head, { childList: true });
console.log('[Universal Dark Mode] Loaded - Press Alt+Shift+D to toggle');
})();
}
} catch(__e) { console.warn('[Userscript:Universal Dark Mode]', __e); }
})();
})();
ARROW-8147: [C++] add GCS library to ThirdpartyToolchain by coryan · Pull Request #11268 · apache/arrow · GitHub
Skip to content
ARROW-8147: [C++] add GCS library to ThirdpartyToolchain#11268

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Closed

coryan
wants to merge
4
commits into

from

 Closed


ARROW-8147: [C++] add GCS library to ThirdpartyToolchain
#11268
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


4 commits


Select commit
Hold shift + click to select a range
a725c52

ARROW-8147: [C++] add GCS library to ThirdpartyToolchain


coryan Sep 29, 2021
da2e245

Address review comments


coryan Sep 30, 2021
4e1341e

Address review comments


coryan Oct 2, 2021
77d5184

Address review comments


coryan Oct 4, 2021

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


2 changes: 2 additions & 0 deletions
ci/docker/ubuntu-20.04-cpp.dockerfile

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -110,6 +110,7 @@ ENV ARROW_BUILD_TESTS=ON \
 ARROW_DATASET=ON \
 ARROW_FLIGHT=OFF \
 ARROW_GANDIVA=ON \
 ARROW_GCS=ON \
 ARROW_HDFS=ON \
 ARROW_HOME=/usr/local \
 ARROW_INSTALL_NAME_RPATH=OFF \
Expand All@@ -129,6 +130,7 @@ ENV ARROW_BUILD_TESTS=ON \
 ARROW_WITH_ZSTD=ON \
 ASAN_SYMBOLIZER_PATH=/usr/lib/llvm-${llvm}/bin/llvm-symbolizer \
 AWSSDK_SOURCE=BUNDLED \
 google_cloud_cpp_storage_SOURCE=BUNDLED \
@pitroupitrouOct 5, 2021

Copy link


Copy Markdown

Member
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Was GCS_SOURCE too ambiguous? This is quite a long parameter name...
@koukouOct 5, 2021

Copy link


Copy Markdown

Member
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We use CMake's package name for XXX_SOURCE.

If we don't follow the convention, we can choose GCS_SOURCE.
@coryancoryanOct 5, 2021

Copy link


Copy Markdown

ContributorAuthor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Let me know how to proceed, I can do either.
@pitroupitrouOct 5, 2021

Copy link


Copy Markdown

Member
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If it's the CMake name then fair enough.
 GTest_SOURCE=BUNDLED \
 gRPC_SOURCE=BUNDLED \
 ORC_SOURCE=BUNDLED \
Expand Down

2 changes: 2 additions & 0 deletions
ci/scripts/cpp_build.sh

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -74,6 +74,7 @@ cmake -G "${CMAKE_GENERATOR:-Ninja}" \
 -DARROW_GANDIVA_JAVA=${ARROW_GANDIVA_JAVA:-OFF} \
 -DARROW_GANDIVA_PC_CXX_FLAGS=${ARROW_GANDIVA_PC_CXX_FLAGS:-} \
 -DARROW_GANDIVA=${ARROW_GANDIVA:-OFF} \
 -DARROW_GCS=${ARROW_GCS:-OFF} \
 -DARROW_HDFS=${ARROW_HDFS:-ON} \
 -DARROW_HIVESERVER2=${ARROW_HIVESERVER2:-OFF} \
 -DARROW_INSTALL_NAME_RPATH=${ARROW_INSTALL_NAME_RPATH:-ON} \
Expand DownExpand Up@@ -121,6 +122,7 @@ cmake -G "${CMAKE_GENERATOR:-Ninja}" \
 -DCMAKE_INSTALL_PREFIX=${CMAKE_INSTALL_PREFIX:-${ARROW_HOME}} \
 -DCMAKE_UNITY_BUILD=${CMAKE_UNITY_BUILD:-OFF} \
 -Dgflags_SOURCE=${gflags_SOURCE:-} \
 -Dgoogle_cloud_cpp_storage_SOURCE=${google_cloud_cpp_storage_SOURCE:-} \
 -DgRPC_SOURCE=${gRPC_SOURCE:-} \
 -DGTest_SOURCE=${GTest_SOURCE:-} \
 -DLz4_SOURCE=${Lz4_SOURCE:-} \
Expand Down

8 changes: 8 additions & 0 deletions
cpp/CMakeLists.txt

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -740,6 +740,14 @@ if(ARROW_ORC)
 endif()
endif()

if(ARROW_GCS)
 list(APPEND ARROW_LINK_LIBS google-cloud-cpp::storage)
 list(APPEND ARROW_STATIC_LINK_LIBS google-cloud-cpp::storage)
 if(google_cloud_cpp_storage_SOURCE STREQUAL "SYSTEM")
 list(APPEND ARROW_STATIC_INSTALL_INTERFACE_LIBS google-cloud-cpp::storage)
 endif()
endif()

if(ARROW_USE_GLOG)
 list(APPEND ARROW_LINK_LIBS glog::glog)
 list(APPEND ARROW_STATIC_LINK_LIBS glog::glog)
Expand Down

4 changes: 4 additions & 0 deletions
cpp/cmake_modules/DefineOptions.cmake

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -224,6 +224,10 @@ if("${CMAKE_SOURCE_DIR}" STREQUAL "${CMAKE_CURRENT_SOURCE_DIR}")

 define_option(ARROW_GANDIVA "Build the Gandiva libraries" OFF)

 define_option(ARROW_GCS
 "Build Arrow with GCS support (requires the GCloud SDK for C++)" OFF)
 mark_as_advanced(ARROW_GCS) # TODO(ARROW-1231) - remove once completed

 define_option(ARROW_HDFS "Build the Arrow HDFS bridge" OFF)

 define_option(ARROW_HIVESERVER2 "Build the HiveServer2 client and Arrow adapter" OFF)
Expand Down

221 changes: 221 additions & 0 deletions
cpp/cmake_modules/ThirdpartyToolchain.cmake

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -151,6 +151,8 @@ macro(build_dependency DEPENDENCY_NAME)
 build_gflags()
 elseif("${DEPENDENCY_NAME}" STREQUAL "GLOG")
 build_glog()
 elseif("${DEPENDENCY_NAME}" STREQUAL "google_cloud_cpp_storage")
 build_google_cloud_cpp_storage()
 elseif("${DEPENDENCY_NAME}" STREQUAL "gRPC")
 build_grpc()
 elseif("${DEPENDENCY_NAME}" STREQUAL "GTest")
Expand DownExpand Up@@ -287,6 +289,10 @@ if(ARROW_FLIGHT)
 set(ARROW_WITH_ZLIB ON)
endif()

if(ARROW_GCS)
 set(ARROW_WITH_GOOGLE_CLOUD_CPP ON)
endif()

if(ARROW_JSON)
 set(ARROW_WITH_RAPIDJSON ON)
endif()
Expand DownExpand Up@@ -433,6 +439,14 @@ else()
 )
endif()

if(DEFINED ENV{ARROW_CRC32C_URL})
 set(CRC32C_URL "$ENV{ARROW_CRC32C_URL}")
else()
 set_urls(CRC32C_SOURCE_URL
 "https://github.com/google/crc32c/archive/${ARROW_CRC32C_BUILD_VERSION}.tar.gz"
 )
endif()

if(DEFINED ENV{ARROW_GBENCHMARK_URL})
 set(GBENCHMARK_SOURCE_URL "$ENV{ARROW_GBENCHMARK_URL}")
else()
Expand DownExpand Up@@ -460,6 +474,14 @@ else()
 )
endif()

if(DEFINED ENV{ARROW_GOOGLE_CLOUD_CPP_URL})
 set(google_cloud_cpp_storage_SOURCE_URL "$ENV{ARROW_GOOGLE_CLOUD_CPP_URL}")
else()
 set_urls(google_cloud_cpp_storage_SOURCE_URL
 "https://github.com/googleapis/google-cloud-cpp/archive/${ARROW_GOOGLE_CLOUD_CPP_BUILD_VERSION}.tar.gz"
 )
endif()

if(DEFINED ENV{ARROW_GRPC_URL})
 set(GRPC_SOURCE_URL "$ENV{ARROW_GRPC_URL}")
else()
Expand DownExpand Up@@ -497,6 +519,14 @@ else()
 )
endif()

if(DEFINED ENV{ARROW_NLOHMANN_JSON_URL})
 set(NLOHMANN_JSON_SOURCE_URL "$ENV{ARROW_NLOHMANN_JSON_URL}")
else()
 set_urls(NLOHMANN_JSON_SOURCE_URL
 "https://github.com/nlohmann/json/archive/${ARROW_NLOHMANN_JSON_BUILD_VERSION}.tar.gz"
 )
endif()

if(DEFINED ENV{ARROW_LZ4_URL})
 set(LZ4_SOURCE_URL "$ENV{ARROW_LZ4_URL}")
else()
Expand DownExpand Up@@ -3470,6 +3500,197 @@ if(ARROW_WITH_GRPC)
 endif()
endif()

# ----------------------------------------------------------------------
# GCS and dependencies

macro(build_crc32c_once)
 if(NOT TARGET crc32c_ep)
 message(STATUS "Building crc32c from source")
 # Build crc32c
 set(CRC32C_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/crc32c_ep-install")
 set(CRC32C_CMAKE_ARGS
 ${EP_COMMON_CMAKE_ARGS}
 -DCMAKE_INSTALL_LIBDIR=lib
 "-DCMAKE_INSTALL_PREFIX=<INSTALL_DIR>"
 -DCMAKE_CXX_STANDARD=11
 -DCRC32C_BUILD_TESTS=OFF
 -DCRC32C_BUILD_BENCHMARKS=OFF
 -DCRC32C_USE_GLOG=OFF)

 set(_CRC32C_STATIC_LIBRARY
 "${CRC32C_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}crc32c${CMAKE_STATIC_LIBRARY_SUFFIX}"
 )
 set(CRC32C_BUILD_BYPRODUCTS ${_CRC32C_STATIC_LIBRARY})
 set(CRC32C_LIBRARIES crc32c)

 externalproject_add(crc32c_ep
 ${EP_LOG_OPTIONS}
 INSTALL_DIR ${CRC32C_PREFIX}
 URL ${CRC32C_SOURCE_URL}
 URL_HASH "SHA256=${ARROW_CRC32C_BUILD_SHA256_CHECKSUM}"
 CMAKE_ARGS ${CRC32C_CMAKE_ARGS}
 BUILD_BYPRODUCTS ${CRC32C_BUILD_BYPRODUCTS})
 add_library(Crc32c::crc32c STATIC IMPORTED)
 set_target_properties(Crc32c::crc32c
 PROPERTIES IMPORTED_LOCATION ${_CRC32C_STATIC_LIBRARY}
 INTERFACE_INCLUDE_DIRECTORIES
 "${CRC32C_INCLUDE}/include")
 add_dependencies(Crc32c::crc32c crc32c_ep)
 endif()
endmacro()

macro(build_nlohmann_json_once)
 if(NOT TARGET nlohmann_json_ep)
 message(STATUS "Building nlohmann-json from source")
 # "Build" nlohmann-json
 set(NLOHMANN_JSON_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/nlohmann_json_ep-install")
 set(NLOHMANN_JSON_CMAKE_ARGS
 ${EP_COMMON_CMAKE_ARGS} -DCMAKE_CXX_STANDARD=11
 "-DCMAKE_INSTALL_PREFIX=<INSTALL_DIR>" -DBUILD_TESTING=OFF)

 set(NLOHMANN_JSON_BUILD_BYPRODUCTS ${NLOHMANN_JSON_PREFIX}/include/json.hpp)

 externalproject_add(nlohmann_json_ep
 ${EP_LOG_OPTIONS}
 INSTALL_DIR ${NLOHMANN_JSON_PREFIX}
 URL ${NLOHMANN_JSON_SOURCE_URL}
 URL_HASH "SHA256=${ARROW_NLOHMANN_JSON_BUILD_SHA256_CHECKSUM}"
 CMAKE_ARGS ${NLOHMANN_JSON_CMAKE_ARGS}
 BUILD_BYPRODUCTS ${NLOHMANN_JSON_BUILD_BYPRODUCTS})
 add_library(nlohmann_json::nlohmann_json INTERFACE IMPORTED)
 set_target_properties(nlohmann_json::nlohmann_json
 PROPERTIES INTERFACE_INCLUDE_DIRECTORIES
 "${NLOHMANN_JSON_PREFIX}/include")
 add_dependencies(nlohmann_json::nlohmann_json nlohmann_json_ep)
 endif()
endmacro()

macro(build_google_cloud_cpp_storage)
 message(STATUS "Building google-cloud-cpp from source")
 message(STATUS "Only building the google-cloud-cpp::storage component")

 # List of dependencies taken from https://github.com/googleapis/google-cloud-cpp/blob/master/doc/packaging.md
 build_absl_once()
 build_crc32c_once()
 build_nlohmann_json_once()

 # Curl is required on all platforms, but building it internally might also trip over S3's copy.
 # For now, force its inclusion from the underlying system or fail.
 find_package(CURL REQUIRED 7.47.0)

 # Build google-cloud-cpp, with only storage_client

 # Inject vendored packages via CMAKE_PREFIX_PATH
 list(APPEND GOOGLE_CLOUD_CPP_PREFIX_PATH_LIST ${ABSL_PREFIX})
 list(APPEND GOOGLE_CLOUD_CPP_PREFIX_PATH_LIST ${CRC32C_PREFIX})
 list(APPEND GOOGLE_CLOUD_CPP_PREFIX_PATH_LIST ${NLOHMANN_JSON_PREFIX})

 set(GOOGLE_CLOUD_CPP_PREFIX_PATH_LIST_SEP_CHAR "|")
 list(JOIN GOOGLE_CLOUD_CPP_PREFIX_PATH_LIST
 ${GOOGLE_CLOUD_CPP_PREFIX_PATH_LIST_SEP_CHAR} GOOGLE_CLOUD_CPP_PREFIX_PATH)

 set(GOOGLE_CLOUD_CPP_INSTALL_PREFIX
 "${CMAKE_CURRENT_BINARY_DIR}/google_cloud_cpp_ep-install")
 set(GOOGLE_CLOUD_CPP_INCLUDE_DIR "${GOOGLE_CLOUD_CPP_INSTALL_PREFIX}/include")
 set(GOOGLE_CLOUD_CPP_CMAKE_ARGS
 ${EP_COMMON_CMAKE_ARGS}
 -DBUILD_TESTING=OFF
 -DCMAKE_INSTALL_LIBDIR=lib
 "-DCMAKE_INSTALL_PREFIX=<INSTALL_DIR>"
 -DCMAKE_INSTALL_RPATH=$ORIGIN
 -DCMAKE_PREFIX_PATH=${GOOGLE_CLOUD_CPP_PREFIX_PATH}
 # Compile only the storage library and its dependencies. To enable
 # other services (Spanner, Bigtable, etc.) add them (as a list) to this
 # parameter. Each has its own `google-cloud-cpp::*` library.
 -DGOOGLE_CLOUD_CPP_ENABLE=storage)
 if(OPENSSL_ROOT_DIR)
 list(APPEND GOOGLE_CLOUD_CPP_CMAKE_ARGS -DOPENSSL_ROOT_DIR=${OPENSSL_ROOT_DIR})
 endif()

 add_custom_target(google_cloud_cpp_dependencies)

 add_dependencies(google_cloud_cpp_dependencies absl_ep)
 add_dependencies(google_cloud_cpp_dependencies crc32c_ep)
 add_dependencies(google_cloud_cpp_dependencies nlohmann_json_ep)
 # Typically the steps to build the AWKSSDK provide `CURL::libcurl`, but if that is
 # disabled we need to provide our own.
 if(NOT TARGET CURL::libcurl)
 find_package(CURL REQUIRED)
 if(NOT TARGET CURL::libcurl)
 # For CMake 3.11 or older
 add_library(CURL::libcurl UNKNOWN IMPORTED)
 set_target_properties(CURL::libcurl
 PROPERTIES INTERFACE_INCLUDE_DIRECTORIES
 "${CURL_INCLUDE_DIRS}" IMPORTED_LOCATION
 "${CURL_LIBRARIES}")
 endif()
 endif()

 set(GOOGLE_CLOUD_CPP_STATIC_LIBRARY_STORAGE
 "${GOOGLE_CLOUD_CPP_INSTALL_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}google_cloud_cpp_storage${CMAKE_STATIC_LIBRARY_SUFFIX}"
 )

 set(GOOGLE_CLOUD_CPP_STATIC_LIBRARY_COMMON
 "${GOOGLE_CLOUD_CPP_INSTALL_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}google_cloud_cpp_common${CMAKE_STATIC_LIBRARY_SUFFIX}"
 )

 externalproject_add(google_cloud_cpp_ep
 ${EP_LOG_OPTIONS}
 LIST_SEPARATOR ${GOOGLE_CLOUD_CPP_PREFIX_PATH_LIST_SEP_CHAR}
 INSTALL_DIR ${GOOGLE_CLOUD_CPP_INSTALL_PREFIX}
 URL ${google_cloud_cpp_storage_SOURCE_URL}
 URL_HASH "SHA256=${ARROW_GOOGLE_CLOUD_CPP_BUILD_SHA256_CHECKSUM}"
 CMAKE_ARGS ${GOOGLE_CLOUD_CPP_CMAKE_ARGS}
 BUILD_BYPRODUCTS ${GOOGLE_CLOUD_CPP_STATIC_LIBRARY_STORAGE}
 ${GOOGLE_CLOUD_CPP_STATIC_LIBRARY_COMMON}
 DEPENDS google_cloud_cpp_dependencies)
 add_dependencies(toolchain google_cloud_cpp_ep)

 add_library(google-cloud-cpp::common STATIC IMPORTED)
 set_target_properties(google-cloud-cpp::common
 PROPERTIES IMPORTED_LOCATION
 "${GOOGLE_CLOUD_CPP_STATIC_LIBRARY_COMMON}"
 INTERFACE_INCLUDE_DIRECTORIES
 "${GOOGLE_CLOUD_CPP_INCLUDE_DIR}")
 set_property(TARGET google-cloud-cpp::common
 PROPERTY INTERFACE_LINK_LIBRARIES
 absl::any
 absl::flat_hash_map
 absl::memory
 absl::optional
 absl::time
 Threads::Threads)

 add_library(google-cloud-cpp::storage STATIC IMPORTED)
 set_target_properties(google-cloud-cpp::storage
 PROPERTIES IMPORTED_LOCATION
 "${GOOGLE_CLOUD_CPP_STATIC_LIBRARY_STORAGE}"
 INTERFACE_INCLUDE_DIRECTORIES
 "${GOOGLE_CLOUD_CPP_INCLUDE_DIR}")
 set_property(TARGET google-cloud-cpp::storage
 PROPERTY INTERFACE_LINK_LIBRARIES
 google-cloud-cpp::common
 absl::memory
 absl::strings
 absl::str_format
 absl::time
 absl::variant
 nlohmann_json::nlohmann_json
 Crc32c::crc32c
 CURL::libcurl
 Threads::Threads
 OpenSSL::SSL
 OpenSSL::Crypto)
 add_dependencies(google-cloud-cpp::storage google_cloud_cpp_ep)

 list(APPEND ARROW_BUNDLED_STATIC_LIBS google-cloud-cpp::storage
 google-cloud-cpp::common)
endmacro()

if(ARROW_WITH_GOOGLE_CLOUD_CPP)
 resolve_dependency(google_cloud_cpp_storage)
endif()

#
# HDFS thirdparty setup

Expand Down

9 changes: 9 additions & 0 deletions
cpp/thirdparty/versions.txt

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -41,12 +41,16 @@ ARROW_BZIP2_BUILD_VERSION=1.0.8
ARROW_BZIP2_BUILD_SHA256_CHECKSUM=ab5a03176ee106d3f0fa90e381da478ddae405918153cca248e682cd0c4a2269
ARROW_CARES_BUILD_VERSION=1.17.1
ARROW_CARES_BUILD_SHA256_CHECKSUM=d73dd0f6de824afd407ce10750ea081af47eba52b8a6cb307d220131ad93fc40
ARROW_CRC32C_BUILD_VERSION=1.1.1
ARROW_CRC32C_BUILD_SHA256_CHECKSUM=a6533f45b1670b5d59b38a514d82b09c6fb70cc1050467220216335e873074e8
ARROW_GBENCHMARK_BUILD_VERSION=v1.5.2
ARROW_GBENCHMARK_BUILD_SHA256_CHECKSUM=dccbdab796baa1043f04982147e67bb6e118fe610da2c65f88912d73987e700c
ARROW_GFLAGS_BUILD_VERSION=v2.2.2
ARROW_GFLAGS_BUILD_SHA256_CHECKSUM=34af2f15cf7367513b352bdcd2493ab14ce43692d2dcd9dfc499492966c64dcf
ARROW_GLOG_BUILD_VERSION=v0.4.0
ARROW_GLOG_BUILD_SHA256_CHECKSUM=f28359aeba12f30d73d9e4711ef356dc842886968112162bc73002645139c39c
ARROW_GOOGLE_CLOUD_CPP_BUILD_VERSION=v1.31.1
ARROW_GOOGLE_CLOUD_CPP_BUILD_SHA256_CHECKSUM=dc7cbf95b506a84b48cf71e0462985d262183edeaabdacaaee2109852394a609
ARROW_GRPC_BUILD_VERSION=v1.35.0
ARROW_GRPC_BUILD_SHA256_CHECKSUM=27dd2fc5c9809ddcde8eb6fa1fa278a3486566dfc28335fca13eb8df8bd3b958
ARROW_GTEST_BUILD_VERSION=1.11.0
Expand All@@ -59,6 +63,8 @@ ARROW_LZ4_BUILD_SHA256_CHECKSUM=030644df4611007ff7dc962d981f390361e6c97a34e5cbc3
# https://github.com/microsoft/mimalloc/issues/353
ARROW_MIMALLOC_BUILD_VERSION=v1.7.2
ARROW_MIMALLOC_BUILD_SHA256_CHECKSUM=b1912e354565a4b698410f7583c0f83934a6dbb3ade54ab7ddcb1569320936bd
ARROW_NLOHMANN_JSON_BUILD_VERSION=v3.10.2
ARROW_NLOHMANN_JSON_BUILD_SHA256_CHECKSUM=081ed0f9f89805c2d96335c3acfa993b39a0a5b4b4cef7edb68dd2210a13458c
ARROW_ORC_BUILD_VERSION=1.7.0
ARROW_ORC_BUILD_SHA256_CHECKSUM=45d6ba9149ffa2aaa168d61ab326f61181861c94529f26da3918a9aa2f801e39
ARROW_PROTOBUF_BUILD_VERSION=v3.17.3
Expand DownExpand Up@@ -98,14 +104,17 @@ DEPENDENCIES=(
 "ARROW_BROTLI_URL brotli-${ARROW_BROTLI_BUILD_VERSION}.tar.gz https://github.com/google/brotli/archive/${ARROW_BROTLI_BUILD_VERSION}.tar.gz"
 "ARROW_BZIP2_URL bzip2-${ARROW_BZIP2_BUILD_VERSION}.tar.gz https://sourceware.org/pub/bzip2/bzip2-${ARROW_BZIP2_BUILD_VERSION}.tar.gz"
 "ARROW_CARES_URL cares-${ARROW_CARES_BUILD_VERSION}.tar.gz https://c-ares.haxx.se/download/c-ares-${ARROW_CARES_BUILD_VERSION}.tar.gz"
 "ARROW_CRC32C_URL crc32c-${ARROW_CRC32C_BUILD_VERSION}.tar.gz https://github.com/google/crc32c/archive/refs/tags/${ARROW_CRC32C_BUILD_VERSION}.tar.gz
 "ARROW_GBENCHMARK_URL gbenchmark-${ARROW_GBENCHMARK_BUILD_VERSION}.tar.gz https://github.com/google/benchmark/archive/${ARROW_GBENCHMARK_BUILD_VERSION}.tar.gz"
 "ARROW_GFLAGS_URL gflags-${ARROW_GFLAGS_BUILD_VERSION}.tar.gz https://github.com/gflags/gflags/archive/${ARROW_GFLAGS_BUILD_VERSION}.tar.gz"
 "ARROW_GLOG_URL glog-${ARROW_GLOG_BUILD_VERSION}.tar.gz https://github.com/google/glog/archive/${ARROW_GLOG_BUILD_VERSION}.tar.gz"
 "ARROW_GOOGLE_CLOUD_CPP_URL google-cloud-cpp-${ARROW_GOOGLE_CLOUD_CPP_BUILD_VERSION}.tar.gz https://github.com/googleapis/google-cloud-cpp/archive/refs/tags/${ARROW_GOOGLE_CLOUD_CPP_BUILD_VERSION}.tar.gz
 "ARROW_GRPC_URL grpc-${ARROW_GRPC_BUILD_VERSION}.tar.gz https://github.com/grpc/grpc/archive/${ARROW_GRPC_BUILD_VERSION}.tar.gz"
 "ARROW_GTEST_URL gtest-${ARROW_GTEST_BUILD_VERSION}.tar.gz https://github.com/google/googletest/archive/release-${ARROW_GTEST_BUILD_VERSION}.tar.gz"
 "ARROW_JEMALLOC_URL jemalloc-${ARROW_JEMALLOC_BUILD_VERSION}.tar.bz2 https://github.com/jemalloc/jemalloc/releases/download/${ARROW_JEMALLOC_BUILD_VERSION}/jemalloc-${ARROW_JEMALLOC_BUILD_VERSION}.tar.bz2"
 "ARROW_LZ4_URL lz4-${ARROW_LZ4_BUILD_VERSION}.tar.gz https://github.com/lz4/lz4/archive/${ARROW_LZ4_BUILD_VERSION}.tar.gz"
 "ARROW_MIMALLOC_URL mimalloc-${ARROW_MIMALLOC_BUILD_VERSION}.tar.gz https://github.com/microsoft/mimalloc/archive/${ARROW_MIMALLOC_BUILD_VERSION}.tar.gz"
 "ARROW_NLOHMANN_JSON_URL nlohmann-json-${ARROW_NLOHMANN_JSON_BUILD_VERSION}.tar.gz https://github.com/nlohmann/json/archive/refs/tags/${ARROW_NLOHMANN_JSON_BUILD_VERSION}.tar.gz
 "ARROW_ORC_URL orc-${ARROW_ORC_BUILD_VERSION}.tar.gz https://github.com/apache/orc/archive/rel/release-${ARROW_ORC_BUILD_VERSION}.tar.gz"
 "ARROW_PROTOBUF_URL protobuf-${ARROW_PROTOBUF_BUILD_VERSION}.tar.gz https://github.com/google/protobuf/releases/download/${ARROW_PROTOBUF_BUILD_VERSION}/protobuf-all-${ARROW_PROTOBUF_BUILD_VERSION:1}.tar.gz"
 "ARROW_RAPIDJSON_URL rapidjson-${ARROW_RAPIDJSON_BUILD_VERSION}.tar.gz https://github.com/miloyip/rapidjson/archive/${ARROW_RAPIDJSON_BUILD_VERSION}.tar.gz"
Expand Down