') + ')', 'gi'); if (regex.test(text)) { found = true; var frag = document.createDocumentFragment(); var parts = text.split(regex); parts.forEach(function(part, i) { if (i % 2 === 0) { frag.appendChild(document.createTextNode(part)); } else { var span = document.createElement('span'); span.className = 'userscript-highlight'; span.textContent = part; frag.appendChild(span); } }); node.parentNode.replaceChild(frag, node); } }); } else if (node.nodeType === 1 && node.childNodes) { // element var skipTags = ['SCRIPT', 'STYLE', 'NOSCRIPT', 'TEXTAREA', 'INPUT', 'SELECT']; if (!skipTags.includes(node.tagName)) { Array.from(node.childNodes).forEach(highlight); } } } highlight(document.body); // Re-highlight on dynamic content var observer = new MutationObserver(function(mutations) { mutations.forEach(function(m) { m.addedNodes.forEach(function(node) { if (node.nodeType === 1 || node.nodeType === 3) highlight(node); }); }); }); observer.observe(document.body, { childList: true, subtree: true }); })(); } } catch(__e) { console.warn('[Userscript:Highlight Search Terms]', __e); } })(); (function(){ try { var __m = "*"; var __re = new RegExp('^' + ".*" + ', 'i'); if (__m === '*' || __re.test(location.href)) { // Strip utm_, fbclid, gclid, etc. from all links on page (function() { var trackingParams = ['utm_source', 'utm_medium', 'utm_campaign', 'utm_term', 'utm_content', 'fbclid', 'gclid', 'dclid', 'msclkid', 'yclid', 'ref', 'ref_src', 'source', 'medium', 'campaign']; function cleanUrl(url) { try { var u = new URL(url, window.location.origin); var changed = false; trackingParams.forEach(function(p) { if (u.searchParams.has(p)) { u.searchParams.delete(p); changed = true; } }); return changed ? u.toString() : url; } catch (e) { return url; } } function cleanLinks() { document.querySelectorAll('a[href]').forEach(function(a) { var clean = cleanUrl(a.href); if (clean !== a.href) a.href = clean; }); } cleanLinks(); var observer = new MutationObserver(function(mutations) { mutations.forEach(function(m) { m.addedNodes.forEach(function(node) { if (node.nodeType === 1) { if (node.tagName === 'A') cleanLinks(); node.querySelectorAll('a[href]').forEach(function(a) { var clean = cleanUrl(a.href); if (clean !== a.href) a.href = clean; }); } }); }); }); observer.observe(document.body, { childList: true, subtree: true }); })(); } } catch(__e) { console.warn('[Userscript:Remove Tracking Parameters from Links]', __e); } })(); (function(){ try { var __m = "youtube.com"; var __re = new RegExp('^' + "youtube\\.com" + ', 'i'); if (__m === '*' || __re.test(location.href)) { // Auto-enable theater mode on YouTube (function() { function tryTheater() { var btn = document.querySelector('button[aria-label="Theater mode"], ytd-player #player button[title="Theater mode"]'); if (btn && !btn.classList.contains('activated')) { btn.click(); } } // Try immediately tryTheater(); // Try after navigation (SPA) var lastUrl = location.href; setInterval(function() { if (location.href !== lastUrl) { lastUrl = location.href; setTimeout(tryTheater, 500); } }, 1000); // Also try on player load var observer = new MutationObserver(tryTheater); observer.observe(document.body, { childList: true, subtree: true }); })(); } } catch(__e) { console.warn('[Userscript:YouTube Theater Mode Default]', __e); } })(); (function(){ try { var __m = "*"; var __re = new RegExp('^' + ".*" + ', 'i'); if (__m === '*' || __re.test(location.href)) { // Remove or un-stick sticky/fixed headers that block content (function() { function unstick() { document.querySelectorAll('header, nav, [role="banner"], .header, .navbar, .sticky, .fixed-top, [style*="position: fixed"], [style*="position:sticky"]').forEach(function(el) { if (el.style.position === 'fixed' || el.style.position === 'sticky' || getComputedStyle(el).position === 'fixed' || getComputedStyle(el).position === 'sticky') { el.style.position = 'static'; el.style.top = 'auto'; el.style.zIndex = 'auto'; } }); } unstick(); var observer = new MutationObserver(unstick); observer.observe(document.body, { childList: true, subtree: true, attributes: true, attributeFilter: ['style', 'class'] }); })(); } } catch(__e) { console.warn('[Userscript:Kill Sticky Headers]', __e); } })(); })(); ARROW-16102: [C++] Add support for building with system gRPC and bundled GCS by kou · Pull Request #12795 · apache/arrow · GitHub
Skip to content

ARROW-16102: [C++] Add support for building with system gRPC and bundled GCS#12795

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
kou wants to merge 9 commits into from
Closed

ARROW-16102: [C++] Add support for building with system gRPC and bundled GCS #12795

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
1a42ed0
ARROW-16102: [C++] Add support for building with system gRPC and bund…
kou Apr 5, 2022
dd052b7
Use google-cloud-cpp 1.39.0
kou Apr 5, 2022
5e93d43
Add OpenSSL::Crypto dependency to google-cloud-cpp::common
kou Apr 5, 2022
611e39a
Enable GCS except non ucrt MinGW
kou Apr 7, 2022
01f6a2c
Use C++ 17 for system Abseil
kou Apr 7, 2022
3688a5c
Use the latest gcs testbench
kou Apr 8, 2022
737084c
Install GCS Testbench on macOS
kou Apr 8, 2022
34fada3
Add missing argument
kou Apr 8, 2022
3e279ab
Use python3
kou Apr 8, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 9 additions & 0 deletions .github/workflows/cpp.yml
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -126,6 +126,7 @@ jobs:
ARROW_DATASET: ON
ARROW_FLIGHT: ON
ARROW_GANDIVA: ON
ARROW_GCS: ON
ARROW_HDFS: ON
ARROW_HOME: /usr/local
ARROW_JEMALLOC: ON
Expand All@@ -141,6 +142,8 @@ jobs:
ARROW_WITH_SNAPPY: ON
ARROW_WITH_ZLIB: ON
ARROW_WITH_ZSTD: ON
# System Abseil installed by Homebrew uses C++ 17
CMAKE_CXX_STANDARD: 17
steps:
- name: Checkout Arrow
uses: actions/checkout@v2
Expand All@@ -153,6 +156,9 @@ jobs:
rm -f /usr/local/bin/2to3
brew update --preinstall
brew bundle --file=cpp/Brewfile
- name: Install Google Cloud Storage Testbench
shell: bash
run: ci/scripts/install_gcs_testbench.sh default
- name: Setup ccache
run: |
ci/scripts/ccache_setup.sh
Expand DownExpand Up@@ -268,6 +274,9 @@ jobs:
ARROW_DATASET: ON
ARROW_FLIGHT: ON
ARROW_GANDIVA: ON
# google-could-cpp uses _dupenv_s() but it can't be used with msvcrt.
# We need to use ucrt to use _dupenv_s().
# ARROW_GCS: ON
ARROW_HDFS: OFF
ARROW_HOME: /mingw${{ matrix.mingw-n-bits }}
ARROW_JEMALLOC: OFF
Expand Down
3 changes: 3 additions & 0 deletions .github/workflows/ruby.yml
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -90,6 +90,7 @@ jobs:
ulimit -c unlimited
archery docker run \
-e ARROW_FLIGHT=ON \
-e ARROW_GCS=ON \
-e Protobuf_SOURCE=BUNDLED \
-e gRPC_SOURCE=BUNDLED \
ubuntu-ruby
Expand All@@ -110,6 +111,7 @@ jobs:
ARROW_BUILD_TESTS: OFF
ARROW_FLIGHT: ON
ARROW_GANDIVA: ON
ARROW_GCS: ON
ARROW_GLIB_GTK_DOC: true
ARROW_GLIB_WERROR: true
ARROW_HOME: /usr/local
Expand DownExpand Up@@ -188,6 +190,7 @@ jobs:
ARROW_BUILD_TYPE: release
ARROW_FLIGHT: ON
ARROW_GANDIVA: ON
ARROW_GCS: ON
ARROW_HDFS: OFF
ARROW_HOME: /ucrt${{ matrix.mingw-n-bits }}
ARROW_JEMALLOC: OFF
Expand Down
5 changes: 3 additions & 2 deletions ci/scripts/install_gcs_testbench.sh
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -31,7 +31,8 @@ fi

version=$1
if [[ "${version}" -eq "default" ]]; then
version="v0.7.0"
version="v0.16.0"
fi

pip install "https://github.com/googleapis/storage-testbench/archive/${version}.tar.gz"
${PYTHON:-python3} -m pip install \
"https://github.com/googleapis/storage-testbench/archive/${version}.tar.gz"
27 changes: 18 additions & 9 deletions cpp/cmake_modules/ThirdpartyToolchain.cmake
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -2593,8 +2593,9 @@ endmacro()
# ----------------------------------------------------------------------
# Dependencies for Arrow Flight RPC

macro(build_absl_once)
if(NOT TARGET absl_ep)
macro(resolve_dependency_absl)
# Choose one of built absl::* targets
if(NOT TARGET absl::algorithm)
message(STATUS "Building Abseil-cpp from source")
set(ABSL_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/absl_ep-install")
set(ABSL_INCLUDE_DIR "${ABSL_PREFIX}/include")
Expand DownExpand Up@@ -3451,6 +3452,7 @@ macro(build_absl_once)
# Work around https://gitlab.kitware.com/cmake/cmake/issues/15052
file(MAKE_DIRECTORY ${ABSL_INCLUDE_DIR})

set(ABSL_VENDORED TRUE)
endif()
endmacro()

Expand All@@ -3464,8 +3466,8 @@ macro(build_grpc)
get_target_property(c-ares_INCLUDE_DIR c-ares::cares INTERFACE_INCLUDE_DIRECTORIES)
include_directories(SYSTEM ${c-ares_INCLUDE_DIR})

# First need to build Abseil
build_absl_once()
# First need Abseil
resolve_dependency_absl()

message(STATUS "Building gRPC from source")

Expand DownExpand Up@@ -3498,7 +3500,9 @@ macro(build_grpc)

add_custom_target(grpc_dependencies)

add_dependencies(grpc_dependencies absl_ep)
if(ABSL_VENDORED)
add_dependencies(grpc_dependencies absl_ep)
endif()
if(CARES_VENDORED)
add_dependencies(grpc_dependencies cares_ep)
endif()
Expand DownExpand Up@@ -3819,7 +3823,7 @@ macro(build_google_cloud_cpp_storage)
message(STATUS "Only building the google-cloud-cpp::storage component")

# List of dependencies taken from https://github.com/googleapis/google-cloud-cpp/blob/master/doc/packaging.md
build_absl_once()
resolve_dependency_absl()
build_crc32c_once()

# Curl is required on all platforms, but building it internally might also trip over S3's copy.
Expand All@@ -3830,7 +3834,9 @@ macro(build_google_cloud_cpp_storage)
# Build google-cloud-cpp, with only storage_client

# Inject vendored packages via CMAKE_PREFIX_PATH
list(APPEND GOOGLE_CLOUD_CPP_PREFIX_PATH_LIST ${ABSL_PREFIX})
if(ABSL_VENDORED)
list(APPEND GOOGLE_CLOUD_CPP_PREFIX_PATH_LIST ${ABSL_PREFIX})
endif()
list(APPEND GOOGLE_CLOUD_CPP_PREFIX_PATH_LIST ${CRC32C_PREFIX})
list(APPEND GOOGLE_CLOUD_CPP_PREFIX_PATH_LIST ${NLOHMANN_JSON_PREFIX})

Expand DownExpand Up@@ -3862,7 +3868,9 @@ macro(build_google_cloud_cpp_storage)

add_custom_target(google_cloud_cpp_dependencies)

add_dependencies(google_cloud_cpp_dependencies absl_ep)
if(ABSL_VENDORED)
add_dependencies(google_cloud_cpp_dependencies absl_ep)
endif()
add_dependencies(google_cloud_cpp_dependencies crc32c_ep)
add_dependencies(google_cloud_cpp_dependencies nlohmann_json::nlohmann_json)

Expand DownExpand Up@@ -3903,7 +3911,8 @@ macro(build_google_cloud_cpp_storage)
absl::memory
absl::optional
absl::time
Threads::Threads)
Threads::Threads
OpenSSL::Crypto)

add_library(google-cloud-cpp::storage STATIC IMPORTED)
set_target_properties(google-cloud-cpp::storage
Expand Down
6 changes: 3 additions & 3 deletions cpp/thirdparty/versions.txt
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -49,8 +49,8 @@ ARROW_GFLAGS_BUILD_VERSION=v2.2.2
ARROW_GFLAGS_BUILD_SHA256_CHECKSUM=34af2f15cf7367513b352bdcd2493ab14ce43692d2dcd9dfc499492966c64dcf
ARROW_GLOG_BUILD_VERSION=v0.5.0
ARROW_GLOG_BUILD_SHA256_CHECKSUM=eede71f28371bf39aa69b45de23b329d37214016e2055269b3b5e7cfd40b59f5
ARROW_GOOGLE_CLOUD_CPP_BUILD_VERSION=v1.35.0
ARROW_GOOGLE_CLOUD_CPP_BUILD_SHA256_CHECKSUM=e4e9eac1e7999eff195db270bc2a719004660b3730ebb5d2f444f2d2057e49b2
ARROW_GOOGLE_CLOUD_CPP_BUILD_VERSION=v1.39.0
ARROW_GOOGLE_CLOUD_CPP_BUILD_SHA256_CHECKSUM=73e4e840018b24bec2beb49e036a3c2d8c471d4dc4a18b9026ccc4d8ab8e78cc
ARROW_GRPC_BUILD_VERSION=v1.35.0
ARROW_GRPC_BUILD_SHA256_CHECKSUM=27dd2fc5c9809ddcde8eb6fa1fa278a3486566dfc28335fca13eb8df8bd3b958
ARROW_GTEST_BUILD_VERSION=1.11.0
Expand DownExpand Up@@ -119,7 +119,7 @@ DEPENDENCIES=(
"ARROW_GBENCHMARK_URL gbenchmark-${ARROW_GBENCHMARK_BUILD_VERSION}.tar.gz https://github.com/google/benchmark/archive/${ARROW_GBENCHMARK_BUILD_VERSION}.tar.gz"
"ARROW_GFLAGS_URL gflags-${ARROW_GFLAGS_BUILD_VERSION}.tar.gz https://github.com/gflags/gflags/archive/${ARROW_GFLAGS_BUILD_VERSION}.tar.gz"
"ARROW_GLOG_URL glog-${ARROW_GLOG_BUILD_VERSION}.tar.gz https://github.com/google/glog/archive/${ARROW_GLOG_BUILD_VERSION}.tar.gz"
"ARROW_GOOGLE_CLOUD_CPP_URL google-cloud-cpp-${ARROW_GOOGLE_CLOUD_CPP_BUILD_VERSION}.tar.gz https://github.com/googleapis/google-cloud-cpp/archive/refs/tags/${ARROW_GOOGLE_CLOUD_CPP_BUILD_VERSION}.tar.gz"
"ARROW_GOOGLE_CLOUD_CPP_URL google-cloud-cpp-${ARROW_GOOGLE_CLOUD_CPP_BUILD_VERSION}.tar.gz https://github.com/googleapis/google-cloud-cpp/archive/${ARROW_GOOGLE_CLOUD_CPP_BUILD_VERSION}.tar.gz"
"ARROW_GRPC_URL grpc-${ARROW_GRPC_BUILD_VERSION}.tar.gz https://github.com/grpc/grpc/archive/${ARROW_GRPC_BUILD_VERSION}.tar.gz"
"ARROW_GTEST_URL gtest-${ARROW_GTEST_BUILD_VERSION}.tar.gz https://github.com/google/googletest/archive/release-${ARROW_GTEST_BUILD_VERSION}.tar.gz"
"ARROW_JEMALLOC_URL jemalloc-${ARROW_JEMALLOC_BUILD_VERSION}.tar.bz2 https://github.com/jemalloc/jemalloc/releases/download/${ARROW_JEMALLOC_BUILD_VERSION}/jemalloc-${ARROW_JEMALLOC_BUILD_VERSION}.tar.bz2"
Expand Down