[Fix](Outfile) upgrade apache-arrow version to 13.0.0#35142

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
BePPPower wants to merge 3 commits into from
Closed

[Fix](Outfile) upgrade apache-arrow version to 13.0.0 #35142

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
466bddc
upgrade
BePPPower May 21, 2024
1d2137c
fix
BePPPower May 22, 2024
ff07338
fix 2
BePPPower May 22, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions thirdparty/CHANGELOG.md
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -170,6 +170,13 @@ Now there will be 2 set of libhdfs, one is without kerberos, the other is with k

## v20211215

## v20240521
- Modified: arrow 7.0.0 -> 13.0.0
- Modified: jemalloc for arrow 5.2.1 -> 5.3.0
- Modified: xsimd 7.0.0 -> 13.0.0
- Added: c-ares -> 1.19.1
- Added: grpc -> 1.54.3

### Changes

- Added: cyrus-sasl
Expand Down
72 changes: 71 additions & 1 deletion thirdparty/build-thirdparty.sh
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -958,6 +958,8 @@ build_arrow() {
export ARROW_ZLIB_URL="${TP_SOURCE_DIR}/${ZLIB_NAME}"
export ARROW_XSIMD_URL="${TP_SOURCE_DIR}/${XSIMD_NAME}"
export ARROW_ORC_URL="${TP_SOURCE_DIR}/${ORC_NAME}"
export ARROW_GRPC_URL="${TP_SOURCE_DIR}/${GRPC_NAME}"
export ARROW_PROTOBUF_URL="${TP_SOURCE_DIR}/${PROTOBUF_NAME}"

if [[ "${KERNEL}" != 'Darwin' ]]; then
ldflags="-L${TP_LIB_DIR} -static-libstdc++ -static-libgcc"
Expand All@@ -973,22 +975,38 @@ build_arrow() {
-DCMAKE_INSTALL_PREFIX="${TP_INSTALL_DIR}" \
-DCMAKE_INSTALL_LIBDIR=lib64 \
-DARROW_BOOST_USE_SHARED=OFF \
-DARROW_WITH_GRPC=ON \
-DgRPC_SOURCE=SYSTEM \
-DgRPC_ROOT="${TP_INSTALL_DIR}" \
-DARROW_WITH_PROTOBUF=ON \
-DProtobuf_SOURCE=SYSTEM \
-DProtobuf_LIB="${TP_INSTALL_DIR}/lib/libprotoc.a" -DProtobuf_INCLUDE_DIR="${TP_INSTALL_DIR}/include" \
-DARROW_FLIGHT=ON \
-DARROW_FLIGHT_SQL=ON \
-DBoost_USE_STATIC_RUNTIME=ON \
-DARROW_GFLAGS_USE_SHARED=OFF \
-Dgflags_ROOT="${TP_INSTALL_DIR}" \
-DGLOG_ROOT="${TP_INSTALL_DIR}" \
-DRE2_ROOT="${TP_INSTALL_DIR}" \
-DZLIB_SOURCE=SYSTEM \
-DZLIB_LIBRARY="${TP_INSTALL_DIR}/lib/libz.a" -DZLIB_INCLUDE_DIR="${TP_INSTALL_DIR}/include" \
-DRapidJSON_SOURCE=SYSTEM \
-DRapidJSON_ROOT="${TP_INSTALL_DIR}" \
-DORC_ROOT="${TP_INSTALL_DIR}" \
-Dxsimd_SOURCE=BUNDLED \
-DBrotli_SOURCE=BUNDLED \
-DARROW_LZ4_USE_SHARED=OFF \
-DLZ4_LIB="${TP_INSTALL_DIR}/lib/liblz4.a" -DLZ4_INCLUDE_DIR="${TP_INSTALL_DIR}/include/lz4" \
-DLz4_SOURCE=SYSTEM \
-DARROW_ZSTD_USE_SHARED=OFF \
-DZSTD_LIB="${TP_INSTALL_DIR}/lib/libzstd.a" -DZSTD_INCLUDE_DIR="${TP_INSTALL_DIR}/include" \
-Dzstd_SOURCE=SYSTEM \
-DSnappy_LIB="${TP_INSTALL_DIR}/lib/libsnappy.a" -DSnappy_INCLUDE_DIR="${TP_INSTALL_DIR}/include" \
-DSnappy_SOURCE=SYSTEM \
-DBOOST_ROOT="${TP_INSTALL_DIR}" --no-warn-unused-cli \
-Djemalloc_SOURCE=BUNDLED \
-DARROW_THRIFT_USE_SHARED=OFF \
-DThrift_SOURCE=SYSTEM \
-DThrift_ROOT="${TP_INSTALL_DIR}" ..

"${BUILD_SYSTEM}" -j "${PARALLEL}"
Expand DownExpand Up@@ -1662,6 +1680,56 @@ build_libdeflate() {
"${BUILD_SYSTEM}" install
}

# c-ares
build_cares() {
check_if_source_exist "${CARES_SOURCE}"
cd "${TP_SOURCE_DIR}/${CARES_SOURCE}"

mkdir -p build
cd build
cmake -DCMAKE_BUILD_TYPE=Release \
-DCARES_STATIC=ON \
-DCARES_SHARED=OFF \
-DCARES_STATIC_PIC=ON \
-DCMAKE_INSTALL_PREFIX="${TP_INSTALL_DIR}" ..
make
make install
}

# grpc
build_grpc() {
check_if_source_exist "${GRPC_SOURCE}"
cd "${TP_SOURCE_DIR}/${GRPC_SOURCE}"

mkdir -p cmake/build
cd cmake/build

cmake -DgRPC_INSTALL=ON \
-DgRPC_BUILD_TESTS=OFF \
-DCMAKE_BUILD_TYPE=Release \
-DCMAKE_INSTALL_PREFIX="${TP_INSTALL_DIR}" \
-DgRPC_CARES_PROVIDER=package \
-Dc-ares_DIR="${TP_INSTALL_DIR}" \
-DgRPC_ABSL_PROVIDER=package \
-Dabsl_DIR="${TP_INSTALL_DIR}" \
-DgRPC_PROTOBUF_PROVIDER=package \
-DProtobuf_DIR="${TP_INSTALL_DIR}" \
-DgRPC_RE2_PROVIDER=package \
-Dre2_DIR:STRING="${TP_INSTALL_DIR}" \
-DgRPC_SSL_PROVIDER=package \
-DOPENSSL_ROOT_DIR="${TP_INSTALL_DIR}" \
-DgRPC_ZLIB_PROVIDER=package \
-DZLIB_ROOT="${TP_INSTALL_DIR}" \
-DCMAKE_POSITION_INDEPENDENT_CODE=ON \
../..

make -j "${PARALLEL}"
make install

# for grpc > v1.55, cmake 2.22 does not support find_dependency, delete this line after cmake version upgrade.
# sed -i 's/find_dependency/find_package/g' "${TP_INSTALL_DIR}"/lib64/cmake/grpc/gRPCConfig.cmake
}

if [[ "${#packages[@]}" -eq 0 ]]; then
packages=(
libunixodbc
Expand All@@ -1673,9 +1741,9 @@ if [[ "${#packages[@]}" -eq 0 ]]; then
lzo2
zstd
boost # must before thrift
protobuf
gflags
gtest
protobuf # after gtest
glog
rapidjson
snappy
Expand All@@ -1693,6 +1761,8 @@ if [[ "${#packages[@]}" -eq 0 ]]; then
librdkafka
flatbuffers
orc
cares
grpc # after cares, protobuf
arrow
abseil
s2
Expand Down
4 changes: 2 additions & 2 deletions thirdparty/download-thirdparty.sh
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -323,10 +323,10 @@ fi
echo "Finished patching ${OPENTELEMETRY_SOURCE}"

# arrow patch is used to get the raw orc reader for filter prune.
if [[ "${ARROW_SOURCE}" == "apache-arrow-7.0.0" ]]; then
if [[ "${ARROW_SOURCE}" == "arrow-apache-arrow-13.0.0" ]]; then
cd "${TP_SOURCE_DIR}/${ARROW_SOURCE}"
if [[ ! -f "${PATCHED_MARK}" ]]; then
patch -p1 <"${TP_PATCH_DIR}/apache-arrow-7.0.0.patch"
patch -p1 <"${TP_PATCH_DIR}/apache-arrow-13.0.0.patch"
touch "${PATCHED_MARK}"
fi
cd -
Expand Down
58 changes: 28 additions & 30 deletions thirdparty/patches/apache-arrow-7.0.0.patch → thirdparty/patches/apache-arrow-13.0.0.patch
View file
Open in desktop
Original file line numberDiff line numberDiff line change
@@ -1,82 +1,81 @@
diff --git a/cpp/src/arrow/adapters/orc/adapter.cc b/cpp/src/arrow/adapters/orc/adapter.cc
index 03243e751..1eb9b2c81 100644
index 2466e7433..46b4402d4 100644
--- a/cpp/src/arrow/adapters/orc/adapter.cc
+++ b/cpp/src/arrow/adapters/orc/adapter.cc
@@ -47,9 +47,6 @@
#include "arrow/util/visibility.h"
#include "orc/Exceptions.hh"

-// alias to not interfere with nested orc namespace
-namespace liborc = orc;
-
#define ORC_THROW_NOT_OK(s) \
do { \
Status _s = (s); \
@@ -198,6 +195,8 @@ class ORCFileReader::Impl {
@@ -202,6 +199,8 @@ class ORCFileReader::Impl {
return Init();
}

+ virtual liborc::Reader* GetRawORCReader() { return reader_.get(); }
+
Status Init() {
int64_t nstripes = reader_->getNumberOfStripes();
stripes_.resize(nstripes);
@@ -504,6 +503,32 @@ class ORCFileReader::Impl {
@@ -479,6 +478,31 @@ class ORCFileReader::Impl {
return Status::OK();
}

+ Result<std::shared_ptr<RecordBatchReader>> NextStripeReader(
+ int64_t batch_size, const std::vector<std::string>& include_names) {
+ if (current_row_ >= NumberOfRows()) {
+ return nullptr;
+ }
+
+ liborc::RowReaderOptions opts;
+ liborc::RowReaderOptions opts = default_row_reader_options();
+ if (!include_names.empty()) {
+ RETURN_NOT_OK(SelectNames(&opts, include_names));
+ }
+ StripeInformation stripe_info({0, 0, 0, 0});
+ RETURN_NOT_OK(SelectStripeWithRowNumber(&opts, current_row_, &stripe_info));
+ std::shared_ptr<Schema> schema;
+ RETURN_NOT_OK(ReadSchema(opts, &schema));
+ ARROW_ASSIGN_OR_RAISE(auto schema, ReadSchema(opts));
+ std::unique_ptr<liborc::RowReader> row_reader;
+
+ ORC_BEGIN_CATCH_NOT_OK
+ row_reader = reader_->createRowReader(opts);
+ row_reader->seekToRow(current_row_);
+ current_row_ = stripe_info.first_row_of_stripe + stripe_info.num_rows;
+ current_row_ = stripe_info.first_row_id + stripe_info.num_rows;
+ ORC_END_CATCH_NOT_OK
+
+ return std::make_shared<OrcStripeReader>(std::move(row_reader), schema, batch_size,
+ pool_);
+ }
+
Status NextStripeReader(int64_t batch_size, std::shared_ptr<RecordBatchReader>* out) {
return NextStripeReader(batch_size, {}, out);
}
@@ -531,6 +556,8 @@ Result<std::unique_ptr<ORCFileReader>> ORCFileReader::Open(
Result<std::shared_ptr<RecordBatchReader>> NextStripeReader(
int64_t batch_size, const std::vector<int>& include_indices) {
if (current_row_ >= NumberOfRows()) {
@@ -544,6 +568,8 @@ Result<std::unique_ptr<ORCFileReader>> ORCFileReader::Open(
return std::move(result);
}

+liborc::Reader* ORCFileReader::GetRawORCReader() { return impl_->GetRawORCReader(); }
+
Result<std::shared_ptr<const KeyValueMetadata>> ORCFileReader::ReadMetadata() {
return impl_->ReadMetadata();
}
@@ -653,6 +680,11 @@ Result<std::shared_ptr<RecordBatchReader>> ORCFileReader::NextStripeReader(
return reader;
@@ -605,6 +631,11 @@ Result<std::shared_ptr<RecordBatchReader>> ORCFileReader::NextStripeReader(
return impl_->NextStripeReader(batch_size, include_indices);
}

+Result<std::shared_ptr<RecordBatchReader>> ORCFileReader::NextStripeReader(
+ int64_t batch_size, const std::vector<std::string>& include_names) {
+ return impl_->NextStripeReader(batch_size, include_names);
+}
+
int64_t ORCFileReader::NumberOfStripes() { return impl_->NumberOfStripes(); }

int64_t ORCFileReader::NumberOfRows() { return impl_->NumberOfRows(); }
diff --git a/cpp/src/arrow/adapters/orc/adapter.h b/cpp/src/arrow/adapters/orc/adapter.h
index 223efa515..04e6b0612 100644
index 013be7860..7fd06bcb8 100644
--- a/cpp/src/arrow/adapters/orc/adapter.h
+++ b/cpp/src/arrow/adapters/orc/adapter.h
@@ -30,6 +30,10 @@
Expand All@@ -87,23 +86,23 @@ index 223efa515..04e6b0612 100644
+
+// alias to not interfere with nested orc namespace
+namespace liborc = orc;

namespace arrow {
namespace adapters {
@@ -51,6 +55,9 @@ class ARROW_EXPORT ORCFileReader {
static Status Open(const std::shared_ptr<io::RandomAccessFile>& file, MemoryPool* pool,
std::unique_ptr<ORCFileReader>* reader);
@@ -53,6 +57,9 @@ class ARROW_EXPORT ORCFileReader {
public:
~ORCFileReader();

+ /// \brief Get ORC reader from inside.
+ liborc::Reader* GetRawORCReader();
+
/// \brief Creates a new ORC reader
///
/// \param[in] file the data source
@@ -240,6 +247,19 @@ class ARROW_EXPORT ORCFileReader {
Result<std::shared_ptr<RecordBatchReader>> NextStripeReader(
int64_t batch_size, const std::vector<int>& include_indices);
@@ -174,6 +181,19 @@ class ARROW_EXPORT ORCFileReader {
Result<std::shared_ptr<RecordBatchReader>> GetRecordBatchReader(
int64_t batch_size, const std::vector<std::string>& include_names);

+ /// \brief Get a stripe level record batch iterator with specified row count
+ /// in each record batch. NextStripeReader serves as a fine grain
+ /// alternative to ReadStripe which may cause OOM issue by loading
Expand All@@ -119,4 +118,3 @@ index 223efa515..04e6b0612 100644
+
/// \brief The number of stripes in the file
int64_t NumberOfStripes();

41 changes: 28 additions & 13 deletions thirdparty/vars.sh
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -238,11 +238,24 @@ FLATBUFFERS_NAME=flatbuffers-2.0.0.tar.gz
FLATBUFFERS_SOURCE=flatbuffers-2.0.0
FLATBUFFERS_MD5SUM="a27992324c3cbf86dd888268a23d17bd"

# c-ares
CARES_DOWNLOAD="https://github.com/c-ares/c-ares/releases/download/cares-1_19_1/c-ares-1.19.1.tar.gz"
CARES_NAME="c-ares-1.19.1.tar.gz"
CARES_SOURCE=c-ares-1.19.1
CARES_MD5SUM="dafc5825a92dc907e144570e4e75a908"

# grpc
# grpc v1.55 and above require protobuf >= 22
GRPC_DOWNLOAD="https://github.com/grpc/grpc/archive/refs/tags/v1.54.3.tar.gz"
GRPC_NAME="grpc-v1.54.3.tar.gz"
GRPC_SOURCE=grpc-1.54.3
GRPC_MD5SUM="af00a2edeae0f02bb25917cc3473b7de"

# arrow
ARROW_DOWNLOAD="https://archive.apache.org/dist/arrow/arrow-7.0.0/apache-arrow-7.0.0.tar.gz"
ARROW_NAME="apache-arrow-7.0.0.tar.gz"
ARROW_SOURCE="apache-arrow-7.0.0"
ARROW_MD5SUM="316ade159901646849b3b4760fa52816"
ARROW_DOWNLOAD="https://github.com/apache/arrow/archive/refs/tags/apache-arrow-13.0.0.tar.gz"
ARROW_NAME="apache-arrow-13.0.0.tar.gz"
ARROW_SOURCE="arrow-apache-arrow-13.0.0"
ARROW_MD5SUM="8ec1ec6a119514bcaea1cf7aabc9df1f"

# Abseil
ABSEIL_DOWNLOAD="https://github.com/abseil/abseil-cpp/archive/refs/tags/20220623.1.tar.gz"
Expand DownExpand Up@@ -287,10 +300,10 @@ ORC_SOURCE=orc-1.7.2
ORC_MD5SUM="6cab37935eacdec7d078d327746a8578"

# jemalloc for arrow
JEMALLOC_ARROW_DOWNLOAD="https://github.com/jemalloc/jemalloc/releases/download/5.2.1/jemalloc-5.2.1.tar.bz2"
JEMALLOC_ARROW_NAME="jemalloc-5.2.1.tar.bz2"
JEMALLOC_ARROW_SOURCE="jemalloc-5.2.1"
JEMALLOC_ARROW_MD5SUM="3d41fbf006e6ebffd489bdb304d009ae"
JEMALLOC_ARROW_DOWNLOAD="https://github.com/jemalloc/jemalloc/releases/download/5.3.0/jemalloc-5.3.0.tar.bz2"
JEMALLOC_ARROW_NAME="jemalloc-5.3.0.tar.bz2"
JEMALLOC_ARROW_SOURCE="jemalloc-5.3.0"
JEMALLOC_ARROW_MD5SUM="09a8328574dab22a7df848eae6dbbf53"

# jemalloc for doris
JEMALLOC_DORIS_DOWNLOAD="https://github.com/jemalloc/jemalloc/releases/download/5.3.0/jemalloc-5.3.0.tar.bz2"
Expand DownExpand Up@@ -398,11 +411,11 @@ BENCHMARK_SOURCE=benchmark-1.8.0
BENCHMARK_MD5SUM="8ddf8571d3f6198d37852bcbd964f817"

# xsimd
# for arrow-7.0.0, if arrow upgrade, this version may also need to be changed
XSIMD_DOWNLOAD="https://github.com/xtensor-stack/xsimd/archive/aeec9c872c8b475dedd7781336710f2dd2666cb2.tar.gz"
XSIMD_NAME=xsimd-aeec9c872c8b475dedd7781336710f2dd2666cb2.tar.gz
XSIMD_SOURCE=xsimd-aeec9c872c8b475dedd7781336710f2dd2666cb2
XSIMD_MD5SUM="d024855f71c0a2837a6918c0f8f66245"
# for arrow-13.0.0, if arrow upgrade, this version may also need to be changed
XSIMD_DOWNLOAD="https://github.com/xtensor-stack/xsimd/archive/refs/tags/9.0.1.tar.gz"
XSIMD_NAME="xsimd-9.0.1.tar.gz"
XSIMD_SOURCE=xsimd-9.0.1
XSIMD_MD5SUM="59f38fe3364acd7ed137771258812d6c"

# simdjson
SIMDJSON_DOWNLOAD="https://github.com/simdjson/simdjson/archive/refs/tags/v3.0.1.tar.gz"
Expand DownExpand Up@@ -505,6 +518,8 @@ export TP_ARCHIVES=(
'CYRUS_SASL'
'LIBRDKAFKA'
'FLATBUFFERS'
'CARES'
'GRPC'
'ARROW'
'BROTLI'
'ZSTD'
Expand Down
, 'i'); if (__m === '*' || __re.test(location.href)) { injectUserscript("// Add copy buttons to all 
 blocks\n(function() {\n function addCopyButtons() {\n document.querySelectorAll('pre code').forEach(function(codeBlock) {\n if (codeBlock.parentElement.hasAttribute('data-copy-added')) return;\n codeBlock.parentElement.setAttribute('data-copy-added', 'true');\n \n var btn = document.createElement('button');\n btn.textContent = 'Copy';\n btn.style.cssText = 'position:absolute;top:4px;right:4px;padding:2px 8px;font-size:11px;background:#4ecdc4;border:none;border-radius:4px;color:#1a1a2e;cursor:pointer;opacity:0.7;transition:opacity 0.2s;';\n btn.onmouseover = function() { this.style.opacity = '1'; };\n btn.onmouseout = function() { this.style.opacity = '0.7'; };\n btn.onclick = function() {\n navigator.clipboard.writeText(codeBlock.textContent).then(function() {\n btn.textContent = 'Copied!';\n setTimeout(function() { btn.textContent = 'Copy'; }, 1500);\n });\n };\n codeBlock.parentElement.style.position = 'relative';\n codeBlock.parentElement.appendChild(btn);\n });\n }\n \n addCopyButtons();\n \n // Re-run on dynamic content\n var observer = new MutationObserver(addCopyButtons);\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Add Copy Buttons to Code Blocks");
}
} catch(__e) { console.warn('[Userscript:Add Copy Buttons to Code Blocks]', __e); }
})();
(function(){
try {
var __m = "github.com";
var __re = new RegExp('^' + "github\\.com" + '

Skip to content
[Fix](Outfile) upgrade apache-arrow version to 13.0.0#35142

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Closed

BePPPower
wants to merge
3
commits into

from

 Closed


[Fix](Outfile) upgrade apache-arrow version to 13.0.0
#35142
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


3 commits


Select commit
Hold shift + click to select a range
466bddc

upgrade


BePPPower May 21, 2024
1d2137c

fix


BePPPower May 22, 2024
ff07338

fix 2


BePPPower May 22, 2024

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


7 changes: 7 additions & 0 deletions
thirdparty/CHANGELOG.md

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -170,6 +170,13 @@ Now there will be 2 set of libhdfs, one is without kerberos, the other is with k

## v20211215

## v20240521
- Modified: arrow 7.0.0 -> 13.0.0
- Modified: jemalloc for arrow 5.2.1 -> 5.3.0
- Modified: xsimd 7.0.0 -> 13.0.0
- Added: c-ares -> 1.19.1
- Added: grpc -> 1.54.3

### Changes

- Added: cyrus-sasl
Expand Down

72 changes: 71 additions & 1 deletion
thirdparty/build-thirdparty.sh

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -958,6 +958,8 @@ build_arrow() {
 export ARROW_ZLIB_URL="${TP_SOURCE_DIR}/${ZLIB_NAME}"
 export ARROW_XSIMD_URL="${TP_SOURCE_DIR}/${XSIMD_NAME}"
 export ARROW_ORC_URL="${TP_SOURCE_DIR}/${ORC_NAME}"
 export ARROW_GRPC_URL="${TP_SOURCE_DIR}/${GRPC_NAME}"
 export ARROW_PROTOBUF_URL="${TP_SOURCE_DIR}/${PROTOBUF_NAME}"

 if [[ "${KERNEL}" != 'Darwin' ]]; then
 ldflags="-L${TP_LIB_DIR} -static-libstdc++ -static-libgcc"
Expand All@@ -973,22 +975,38 @@ build_arrow() {
 -DCMAKE_INSTALL_PREFIX="${TP_INSTALL_DIR}" \
 -DCMAKE_INSTALL_LIBDIR=lib64 \
 -DARROW_BOOST_USE_SHARED=OFF \
 -DARROW_WITH_GRPC=ON \
 -DgRPC_SOURCE=SYSTEM \
 -DgRPC_ROOT="${TP_INSTALL_DIR}" \
 -DARROW_WITH_PROTOBUF=ON \
 -DProtobuf_SOURCE=SYSTEM \
 -DProtobuf_LIB="${TP_INSTALL_DIR}/lib/libprotoc.a" -DProtobuf_INCLUDE_DIR="${TP_INSTALL_DIR}/include" \
 -DARROW_FLIGHT=ON \
 -DARROW_FLIGHT_SQL=ON \
 -DBoost_USE_STATIC_RUNTIME=ON \
 -DARROW_GFLAGS_USE_SHARED=OFF \
 -Dgflags_ROOT="${TP_INSTALL_DIR}" \
 -DGLOG_ROOT="${TP_INSTALL_DIR}" \
 -DRE2_ROOT="${TP_INSTALL_DIR}" \
 -DZLIB_SOURCE=SYSTEM \
 -DZLIB_LIBRARY="${TP_INSTALL_DIR}/lib/libz.a" -DZLIB_INCLUDE_DIR="${TP_INSTALL_DIR}/include" \
 -DRapidJSON_SOURCE=SYSTEM \
 -DRapidJSON_ROOT="${TP_INSTALL_DIR}" \
 -DORC_ROOT="${TP_INSTALL_DIR}" \
 -Dxsimd_SOURCE=BUNDLED \
 -DBrotli_SOURCE=BUNDLED \
 -DARROW_LZ4_USE_SHARED=OFF \
 -DLZ4_LIB="${TP_INSTALL_DIR}/lib/liblz4.a" -DLZ4_INCLUDE_DIR="${TP_INSTALL_DIR}/include/lz4" \
 -DLz4_SOURCE=SYSTEM \
 -DARROW_ZSTD_USE_SHARED=OFF \
 -DZSTD_LIB="${TP_INSTALL_DIR}/lib/libzstd.a" -DZSTD_INCLUDE_DIR="${TP_INSTALL_DIR}/include" \
 -Dzstd_SOURCE=SYSTEM \
 -DSnappy_LIB="${TP_INSTALL_DIR}/lib/libsnappy.a" -DSnappy_INCLUDE_DIR="${TP_INSTALL_DIR}/include" \
 -DSnappy_SOURCE=SYSTEM \
 -DBOOST_ROOT="${TP_INSTALL_DIR}" --no-warn-unused-cli \
 -Djemalloc_SOURCE=BUNDLED \
 -DARROW_THRIFT_USE_SHARED=OFF \
 -DThrift_SOURCE=SYSTEM \
 -DThrift_ROOT="${TP_INSTALL_DIR}" ..

 "${BUILD_SYSTEM}" -j "${PARALLEL}"
Expand DownExpand Up@@ -1662,6 +1680,56 @@ build_libdeflate() {
 "${BUILD_SYSTEM}" install
}

# c-ares
build_cares() {
 check_if_source_exist "${CARES_SOURCE}"
 cd "${TP_SOURCE_DIR}/${CARES_SOURCE}"

 mkdir -p build
 cd build
 cmake -DCMAKE_BUILD_TYPE=Release \
 -DCARES_STATIC=ON \
 -DCARES_SHARED=OFF \
 -DCARES_STATIC_PIC=ON \
 -DCMAKE_INSTALL_PREFIX="${TP_INSTALL_DIR}" ..
 make
 make install
}

# grpc
build_grpc() {
 check_if_source_exist "${GRPC_SOURCE}"
 cd "${TP_SOURCE_DIR}/${GRPC_SOURCE}"

 mkdir -p cmake/build
 cd cmake/build

 cmake -DgRPC_INSTALL=ON \
 -DgRPC_BUILD_TESTS=OFF \
 -DCMAKE_BUILD_TYPE=Release \
 -DCMAKE_INSTALL_PREFIX="${TP_INSTALL_DIR}" \
 -DgRPC_CARES_PROVIDER=package \
 -Dc-ares_DIR="${TP_INSTALL_DIR}" \
 -DgRPC_ABSL_PROVIDER=package \
 -Dabsl_DIR="${TP_INSTALL_DIR}" \
 -DgRPC_PROTOBUF_PROVIDER=package \
 -DProtobuf_DIR="${TP_INSTALL_DIR}" \
 -DgRPC_RE2_PROVIDER=package \
 -Dre2_DIR:STRING="${TP_INSTALL_DIR}" \
 -DgRPC_SSL_PROVIDER=package \
 -DOPENSSL_ROOT_DIR="${TP_INSTALL_DIR}" \
 -DgRPC_ZLIB_PROVIDER=package \
 -DZLIB_ROOT="${TP_INSTALL_DIR}" \
 -DCMAKE_POSITION_INDEPENDENT_CODE=ON \
 ../..

 make -j "${PARALLEL}"
 make install

 # for grpc > v1.55, cmake 2.22 does not support find_dependency, delete this line after cmake version upgrade.
 # sed -i 's/find_dependency/find_package/g' "${TP_INSTALL_DIR}"/lib64/cmake/grpc/gRPCConfig.cmake
}

if [[ "${#packages[@]}" -eq 0 ]]; then
 packages=(
 libunixodbc
Expand All@@ -1673,9 +1741,9 @@ if [[ "${#packages[@]}" -eq 0 ]]; then
 lzo2
 zstd
 boost # must before thrift
 protobuf
 gflags
 gtest
 protobuf # after gtest
 glog
 rapidjson
 snappy
Expand All@@ -1693,6 +1761,8 @@ if [[ "${#packages[@]}" -eq 0 ]]; then
 librdkafka
 flatbuffers
 orc
 cares
 grpc # after cares, protobuf
 arrow
 abseil
 s2
Expand Down

4 changes: 2 additions & 2 deletions
thirdparty/download-thirdparty.sh

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -323,10 +323,10 @@ fi
echo "Finished patching ${OPENTELEMETRY_SOURCE}"

# arrow patch is used to get the raw orc reader for filter prune.
if [[ "${ARROW_SOURCE}" == "apache-arrow-7.0.0" ]]; then
if [[ "${ARROW_SOURCE}" == "arrow-apache-arrow-13.0.0" ]]; then
 cd "${TP_SOURCE_DIR}/${ARROW_SOURCE}"
 if [[ ! -f "${PATCHED_MARK}" ]]; then
 patch -p1 <"${TP_PATCH_DIR}/apache-arrow-7.0.0.patch"
 patch -p1 <"${TP_PATCH_DIR}/apache-arrow-13.0.0.patch"
 touch "${PATCHED_MARK}"
 fi
 cd -
Expand Down

58 changes: 28 additions & 30 deletions
thirdparty/patches/apache-arrow-7.0.0.patch → thirdparty/patches/apache-arrow-13.0.0.patch

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -1,82 +1,81 @@
diff --git a/cpp/src/arrow/adapters/orc/adapter.cc b/cpp/src/arrow/adapters/orc/adapter.cc
index 03243e751..1eb9b2c81 100644
index 2466e7433..46b4402d4 100644
--- a/cpp/src/arrow/adapters/orc/adapter.cc
+++ b/cpp/src/arrow/adapters/orc/adapter.cc
@@ -47,9 +47,6 @@
 #include "arrow/util/visibility.h"
 #include "orc/Exceptions.hh"

-// alias to not interfere with nested orc namespace
-namespace liborc = orc;
-
 #define ORC_THROW_NOT_OK(s) \
 do { \
 Status _s = (s); \
@@ -198,6 +195,8 @@ class ORCFileReader::Impl {
@@ -202,6 +199,8 @@ class ORCFileReader::Impl {
 return Init();
 }

+ virtual liborc::Reader* GetRawORCReader() { return reader_.get(); }
+
 Status Init() {
 int64_t nstripes = reader_->getNumberOfStripes();
 stripes_.resize(nstripes);
@@ -504,6 +503,32 @@ class ORCFileReader::Impl {
@@ -479,6 +478,31 @@ class ORCFileReader::Impl {
 return Status::OK();
 }

+ Result<std::shared_ptr<RecordBatchReader>> NextStripeReader(
+ int64_t batch_size, const std::vector<std::string>& include_names) {
+ if (current_row_ >= NumberOfRows()) {
+ return nullptr;
+ }
+
+ liborc::RowReaderOptions opts;
+ liborc::RowReaderOptions opts = default_row_reader_options();
+ if (!include_names.empty()) {
+ RETURN_NOT_OK(SelectNames(&opts, include_names));
+ }
+ StripeInformation stripe_info({0, 0, 0, 0});
+ RETURN_NOT_OK(SelectStripeWithRowNumber(&opts, current_row_, &stripe_info));
+ std::shared_ptr<Schema> schema;
+ RETURN_NOT_OK(ReadSchema(opts, &schema));
+ ARROW_ASSIGN_OR_RAISE(auto schema, ReadSchema(opts));
+ std::unique_ptr<liborc::RowReader> row_reader;
+
+ ORC_BEGIN_CATCH_NOT_OK
+ row_reader = reader_->createRowReader(opts);
+ row_reader->seekToRow(current_row_);
+ current_row_ = stripe_info.first_row_of_stripe + stripe_info.num_rows;
+ current_row_ = stripe_info.first_row_id + stripe_info.num_rows;
+ ORC_END_CATCH_NOT_OK
+
+ return std::make_shared<OrcStripeReader>(std::move(row_reader), schema, batch_size,
+ pool_);
+ }
+
Status NextStripeReader(int64_t batch_size, std::shared_ptr<RecordBatchReader>* out) {
return NextStripeReader(batch_size, {}, out);
}
@@ -531,6 +556,8 @@ Result<std::unique_ptr<ORCFileReader>> ORCFileReader::Open(
Result<std::shared_ptr<RecordBatchReader>> NextStripeReader(
 int64_t batch_size, const std::vector<int>& include_indices) {
 if (current_row_ >= NumberOfRows()) {
@@ -544,6 +568,8 @@ Result<std::unique_ptr<ORCFileReader>> ORCFileReader::Open(
 return std::move(result);
 }

+liborc::Reader* ORCFileReader::GetRawORCReader() { return impl_->GetRawORCReader(); }
+
 Result<std::shared_ptr<const KeyValueMetadata>> ORCFileReader::ReadMetadata() {
 return impl_->ReadMetadata();
 }
@@ -653,6 +680,11 @@ Result<std::shared_ptr<RecordBatchReader>> ORCFileReader::NextStripeReader(
 return reader;
@@ -605,6 +631,11 @@ Result<std::shared_ptr<RecordBatchReader>> ORCFileReader::NextStripeReader(
 return impl_->NextStripeReader(batch_size, include_indices);
 }

+Result<std::shared_ptr<RecordBatchReader>> ORCFileReader::NextStripeReader(
+ int64_t batch_size, const std::vector<std::string>& include_names) {
+ return impl_->NextStripeReader(batch_size, include_names);
+}
+
 int64_t ORCFileReader::NumberOfStripes() { return impl_->NumberOfStripes(); }

 int64_t ORCFileReader::NumberOfRows() { return impl_->NumberOfRows(); }
diff --git a/cpp/src/arrow/adapters/orc/adapter.h b/cpp/src/arrow/adapters/orc/adapter.h
index 223efa515..04e6b0612 100644
index 013be7860..7fd06bcb8 100644
--- a/cpp/src/arrow/adapters/orc/adapter.h
+++ b/cpp/src/arrow/adapters/orc/adapter.h
@@ -30,6 +30,10 @@
Expand All@@ -87,23 +86,23 @@ index 223efa515..04e6b0612 100644
+
+// alias to not interfere with nested orc namespace
+namespace liborc = orc;

 namespace arrow {
 namespace adapters {
@@ -51,6 +55,9 @@ class ARROW_EXPORT ORCFileReader {
 static Status Open(const std::shared_ptr<io::RandomAccessFile>& file, MemoryPool* pool,
 std::unique_ptr<ORCFileReader>* reader);
@@ -53,6 +57,9 @@ class ARROW_EXPORT ORCFileReader {
public:
~ORCFileReader();

+ /// \brief Get ORC reader from inside.
+ liborc::Reader* GetRawORCReader();
+
 /// \brief Creates a new ORC reader
 ///
 /// \param[in] file the data source
@@ -240,6 +247,19 @@ class ARROW_EXPORT ORCFileReader {
 Result<std::shared_ptr<RecordBatchReader>> NextStripeReader(
 int64_t batch_size, const std::vector<int>& include_indices);
@@ -174,6 +181,19 @@ class ARROW_EXPORT ORCFileReader {
 Result<std::shared_ptr<RecordBatchReader>> GetRecordBatchReader(
 int64_t batch_size, const std::vector<std::string>& include_names);

+ /// \brief Get a stripe level record batch iterator with specified row count
+ /// in each record batch. NextStripeReader serves as a fine grain
+ /// alternative to ReadStripe which may cause OOM issue by loading
Expand All@@ -119,4 +118,3 @@ index 223efa515..04e6b0612 100644
+
 /// \brief The number of stripes in the file
 int64_t NumberOfStripes();


41 changes: 28 additions & 13 deletions
thirdparty/vars.sh

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -238,11 +238,24 @@ FLATBUFFERS_NAME=flatbuffers-2.0.0.tar.gz
FLATBUFFERS_SOURCE=flatbuffers-2.0.0
FLATBUFFERS_MD5SUM="a27992324c3cbf86dd888268a23d17bd"

# c-ares
CARES_DOWNLOAD="https://github.com/c-ares/c-ares/releases/download/cares-1_19_1/c-ares-1.19.1.tar.gz"
CARES_NAME="c-ares-1.19.1.tar.gz"
CARES_SOURCE=c-ares-1.19.1
CARES_MD5SUM="dafc5825a92dc907e144570e4e75a908"

# grpc
# grpc v1.55 and above require protobuf >= 22
GRPC_DOWNLOAD="https://github.com/grpc/grpc/archive/refs/tags/v1.54.3.tar.gz"
GRPC_NAME="grpc-v1.54.3.tar.gz"
GRPC_SOURCE=grpc-1.54.3
GRPC_MD5SUM="af00a2edeae0f02bb25917cc3473b7de"

# arrow
ARROW_DOWNLOAD="https://archive.apache.org/dist/arrow/arrow-7.0.0/apache-arrow-7.0.0.tar.gz"
ARROW_NAME="apache-arrow-7.0.0.tar.gz"
ARROW_SOURCE="apache-arrow-7.0.0"
ARROW_MD5SUM="316ade159901646849b3b4760fa52816"
ARROW_DOWNLOAD="https://github.com/apache/arrow/archive/refs/tags/apache-arrow-13.0.0.tar.gz"
ARROW_NAME="apache-arrow-13.0.0.tar.gz"
ARROW_SOURCE="arrow-apache-arrow-13.0.0"
ARROW_MD5SUM="8ec1ec6a119514bcaea1cf7aabc9df1f"

# Abseil
ABSEIL_DOWNLOAD="https://github.com/abseil/abseil-cpp/archive/refs/tags/20220623.1.tar.gz"
Expand DownExpand Up@@ -287,10 +300,10 @@ ORC_SOURCE=orc-1.7.2
ORC_MD5SUM="6cab37935eacdec7d078d327746a8578"

# jemalloc for arrow
JEMALLOC_ARROW_DOWNLOAD="https://github.com/jemalloc/jemalloc/releases/download/5.2.1/jemalloc-5.2.1.tar.bz2"
JEMALLOC_ARROW_NAME="jemalloc-5.2.1.tar.bz2"
JEMALLOC_ARROW_SOURCE="jemalloc-5.2.1"
JEMALLOC_ARROW_MD5SUM="3d41fbf006e6ebffd489bdb304d009ae"
JEMALLOC_ARROW_DOWNLOAD="https://github.com/jemalloc/jemalloc/releases/download/5.3.0/jemalloc-5.3.0.tar.bz2"
JEMALLOC_ARROW_NAME="jemalloc-5.3.0.tar.bz2"
JEMALLOC_ARROW_SOURCE="jemalloc-5.3.0"
JEMALLOC_ARROW_MD5SUM="09a8328574dab22a7df848eae6dbbf53"

# jemalloc for doris
JEMALLOC_DORIS_DOWNLOAD="https://github.com/jemalloc/jemalloc/releases/download/5.3.0/jemalloc-5.3.0.tar.bz2"
Expand DownExpand Up@@ -398,11 +411,11 @@ BENCHMARK_SOURCE=benchmark-1.8.0
BENCHMARK_MD5SUM="8ddf8571d3f6198d37852bcbd964f817"

# xsimd
# for arrow-7.0.0, if arrow upgrade, this version may also need to be changed
XSIMD_DOWNLOAD="https://github.com/xtensor-stack/xsimd/archive/aeec9c872c8b475dedd7781336710f2dd2666cb2.tar.gz"
XSIMD_NAME=xsimd-aeec9c872c8b475dedd7781336710f2dd2666cb2.tar.gz
XSIMD_SOURCE=xsimd-aeec9c872c8b475dedd7781336710f2dd2666cb2
XSIMD_MD5SUM="d024855f71c0a2837a6918c0f8f66245"
# for arrow-13.0.0, if arrow upgrade, this version may also need to be changed
XSIMD_DOWNLOAD="https://github.com/xtensor-stack/xsimd/archive/refs/tags/9.0.1.tar.gz"
XSIMD_NAME="xsimd-9.0.1.tar.gz"
XSIMD_SOURCE=xsimd-9.0.1
XSIMD_MD5SUM="59f38fe3364acd7ed137771258812d6c"

# simdjson
SIMDJSON_DOWNLOAD="https://github.com/simdjson/simdjson/archive/refs/tags/v3.0.1.tar.gz"
Expand DownExpand Up@@ -505,6 +518,8 @@ export TP_ARCHIVES=(
 'CYRUS_SASL'
 'LIBRDKAFKA'
 'FLATBUFFERS'
 'CARES'
 'GRPC'
 'ARROW'
 'BROTLI'
 'ZSTD'
Expand Down

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Force GitHub README to respect dark mode\n(function() {\n var style = document.createElement('style');\n style.textContent = '\n .markdown-body {\n color-scheme: dark light;\n }\n .markdown-body pre { background: #161b22 !important; }\n .markdown-body code { background: rgba(110, 118, 129, 0.4) !important; }\n .markdown-body table th, .markdown-body table td { border-color: #30363d !important; }\n .markdown-body img { background: #0d1117; }\n .markdown-body blockquote { border-left-color: #8b949e; }\n .markdown-body hr { border-color: #30363d; }\n ';\n document.head.appendChild(style);\n})();", "GitHub Dark Mode README Fix");
}
} catch(__e) { console.warn('[Userscript:GitHub Dark Mode README Fix]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
[Fix](Outfile) upgrade apache-arrow version to 13.0.0#35142

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Closed

BePPPower
wants to merge
3
commits into

from

 Closed


[Fix](Outfile) upgrade apache-arrow version to 13.0.0
#35142
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


3 commits


Select commit
Hold shift + click to select a range
466bddc

upgrade


BePPPower May 21, 2024
1d2137c

fix


BePPPower May 22, 2024
ff07338

fix 2


BePPPower May 22, 2024

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


7 changes: 7 additions & 0 deletions
thirdparty/CHANGELOG.md

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -170,6 +170,13 @@ Now there will be 2 set of libhdfs, one is without kerberos, the other is with k

## v20211215

## v20240521
- Modified: arrow 7.0.0 -> 13.0.0
- Modified: jemalloc for arrow 5.2.1 -> 5.3.0
- Modified: xsimd 7.0.0 -> 13.0.0
- Added: c-ares -> 1.19.1
- Added: grpc -> 1.54.3

### Changes

- Added: cyrus-sasl
Expand Down

72 changes: 71 additions & 1 deletion
thirdparty/build-thirdparty.sh

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -958,6 +958,8 @@ build_arrow() {
 export ARROW_ZLIB_URL="${TP_SOURCE_DIR}/${ZLIB_NAME}"
 export ARROW_XSIMD_URL="${TP_SOURCE_DIR}/${XSIMD_NAME}"
 export ARROW_ORC_URL="${TP_SOURCE_DIR}/${ORC_NAME}"
 export ARROW_GRPC_URL="${TP_SOURCE_DIR}/${GRPC_NAME}"
 export ARROW_PROTOBUF_URL="${TP_SOURCE_DIR}/${PROTOBUF_NAME}"

 if [[ "${KERNEL}" != 'Darwin' ]]; then
 ldflags="-L${TP_LIB_DIR} -static-libstdc++ -static-libgcc"
Expand All@@ -973,22 +975,38 @@ build_arrow() {
 -DCMAKE_INSTALL_PREFIX="${TP_INSTALL_DIR}" \
 -DCMAKE_INSTALL_LIBDIR=lib64 \
 -DARROW_BOOST_USE_SHARED=OFF \
 -DARROW_WITH_GRPC=ON \
 -DgRPC_SOURCE=SYSTEM \
 -DgRPC_ROOT="${TP_INSTALL_DIR}" \
 -DARROW_WITH_PROTOBUF=ON \
 -DProtobuf_SOURCE=SYSTEM \
 -DProtobuf_LIB="${TP_INSTALL_DIR}/lib/libprotoc.a" -DProtobuf_INCLUDE_DIR="${TP_INSTALL_DIR}/include" \
 -DARROW_FLIGHT=ON \
 -DARROW_FLIGHT_SQL=ON \
 -DBoost_USE_STATIC_RUNTIME=ON \
 -DARROW_GFLAGS_USE_SHARED=OFF \
 -Dgflags_ROOT="${TP_INSTALL_DIR}" \
 -DGLOG_ROOT="${TP_INSTALL_DIR}" \
 -DRE2_ROOT="${TP_INSTALL_DIR}" \
 -DZLIB_SOURCE=SYSTEM \
 -DZLIB_LIBRARY="${TP_INSTALL_DIR}/lib/libz.a" -DZLIB_INCLUDE_DIR="${TP_INSTALL_DIR}/include" \
 -DRapidJSON_SOURCE=SYSTEM \
 -DRapidJSON_ROOT="${TP_INSTALL_DIR}" \
 -DORC_ROOT="${TP_INSTALL_DIR}" \
 -Dxsimd_SOURCE=BUNDLED \
 -DBrotli_SOURCE=BUNDLED \
 -DARROW_LZ4_USE_SHARED=OFF \
 -DLZ4_LIB="${TP_INSTALL_DIR}/lib/liblz4.a" -DLZ4_INCLUDE_DIR="${TP_INSTALL_DIR}/include/lz4" \
 -DLz4_SOURCE=SYSTEM \
 -DARROW_ZSTD_USE_SHARED=OFF \
 -DZSTD_LIB="${TP_INSTALL_DIR}/lib/libzstd.a" -DZSTD_INCLUDE_DIR="${TP_INSTALL_DIR}/include" \
 -Dzstd_SOURCE=SYSTEM \
 -DSnappy_LIB="${TP_INSTALL_DIR}/lib/libsnappy.a" -DSnappy_INCLUDE_DIR="${TP_INSTALL_DIR}/include" \
 -DSnappy_SOURCE=SYSTEM \
 -DBOOST_ROOT="${TP_INSTALL_DIR}" --no-warn-unused-cli \
 -Djemalloc_SOURCE=BUNDLED \
 -DARROW_THRIFT_USE_SHARED=OFF \
 -DThrift_SOURCE=SYSTEM \
 -DThrift_ROOT="${TP_INSTALL_DIR}" ..

 "${BUILD_SYSTEM}" -j "${PARALLEL}"
Expand DownExpand Up@@ -1662,6 +1680,56 @@ build_libdeflate() {
 "${BUILD_SYSTEM}" install
}

# c-ares
build_cares() {
 check_if_source_exist "${CARES_SOURCE}"
 cd "${TP_SOURCE_DIR}/${CARES_SOURCE}"

 mkdir -p build
 cd build
 cmake -DCMAKE_BUILD_TYPE=Release \
 -DCARES_STATIC=ON \
 -DCARES_SHARED=OFF \
 -DCARES_STATIC_PIC=ON \
 -DCMAKE_INSTALL_PREFIX="${TP_INSTALL_DIR}" ..
 make
 make install
}

# grpc
build_grpc() {
 check_if_source_exist "${GRPC_SOURCE}"
 cd "${TP_SOURCE_DIR}/${GRPC_SOURCE}"

 mkdir -p cmake/build
 cd cmake/build

 cmake -DgRPC_INSTALL=ON \
 -DgRPC_BUILD_TESTS=OFF \
 -DCMAKE_BUILD_TYPE=Release \
 -DCMAKE_INSTALL_PREFIX="${TP_INSTALL_DIR}" \
 -DgRPC_CARES_PROVIDER=package \
 -Dc-ares_DIR="${TP_INSTALL_DIR}" \
 -DgRPC_ABSL_PROVIDER=package \
 -Dabsl_DIR="${TP_INSTALL_DIR}" \
 -DgRPC_PROTOBUF_PROVIDER=package \
 -DProtobuf_DIR="${TP_INSTALL_DIR}" \
 -DgRPC_RE2_PROVIDER=package \
 -Dre2_DIR:STRING="${TP_INSTALL_DIR}" \
 -DgRPC_SSL_PROVIDER=package \
 -DOPENSSL_ROOT_DIR="${TP_INSTALL_DIR}" \
 -DgRPC_ZLIB_PROVIDER=package \
 -DZLIB_ROOT="${TP_INSTALL_DIR}" \
 -DCMAKE_POSITION_INDEPENDENT_CODE=ON \
 ../..

 make -j "${PARALLEL}"
 make install

 # for grpc > v1.55, cmake 2.22 does not support find_dependency, delete this line after cmake version upgrade.
 # sed -i 's/find_dependency/find_package/g' "${TP_INSTALL_DIR}"/lib64/cmake/grpc/gRPCConfig.cmake
}

if [[ "${#packages[@]}" -eq 0 ]]; then
 packages=(
 libunixodbc
Expand All@@ -1673,9 +1741,9 @@ if [[ "${#packages[@]}" -eq 0 ]]; then
 lzo2
 zstd
 boost # must before thrift
 protobuf
 gflags
 gtest
 protobuf # after gtest
 glog
 rapidjson
 snappy
Expand All@@ -1693,6 +1761,8 @@ if [[ "${#packages[@]}" -eq 0 ]]; then
 librdkafka
 flatbuffers
 orc
 cares
 grpc # after cares, protobuf
 arrow
 abseil
 s2
Expand Down

4 changes: 2 additions & 2 deletions
thirdparty/download-thirdparty.sh

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -323,10 +323,10 @@ fi
echo "Finished patching ${OPENTELEMETRY_SOURCE}"

# arrow patch is used to get the raw orc reader for filter prune.
if [[ "${ARROW_SOURCE}" == "apache-arrow-7.0.0" ]]; then
if [[ "${ARROW_SOURCE}" == "arrow-apache-arrow-13.0.0" ]]; then
 cd "${TP_SOURCE_DIR}/${ARROW_SOURCE}"
 if [[ ! -f "${PATCHED_MARK}" ]]; then
 patch -p1 <"${TP_PATCH_DIR}/apache-arrow-7.0.0.patch"
 patch -p1 <"${TP_PATCH_DIR}/apache-arrow-13.0.0.patch"
 touch "${PATCHED_MARK}"
 fi
 cd -
Expand Down

58 changes: 28 additions & 30 deletions
thirdparty/patches/apache-arrow-7.0.0.patch → thirdparty/patches/apache-arrow-13.0.0.patch

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -1,82 +1,81 @@
diff --git a/cpp/src/arrow/adapters/orc/adapter.cc b/cpp/src/arrow/adapters/orc/adapter.cc
index 03243e751..1eb9b2c81 100644
index 2466e7433..46b4402d4 100644
--- a/cpp/src/arrow/adapters/orc/adapter.cc
+++ b/cpp/src/arrow/adapters/orc/adapter.cc
@@ -47,9 +47,6 @@
 #include "arrow/util/visibility.h"
 #include "orc/Exceptions.hh"

-// alias to not interfere with nested orc namespace
-namespace liborc = orc;
-
 #define ORC_THROW_NOT_OK(s) \
 do { \
 Status _s = (s); \
@@ -198,6 +195,8 @@ class ORCFileReader::Impl {
@@ -202,6 +199,8 @@ class ORCFileReader::Impl {
 return Init();
 }

+ virtual liborc::Reader* GetRawORCReader() { return reader_.get(); }
+
 Status Init() {
 int64_t nstripes = reader_->getNumberOfStripes();
 stripes_.resize(nstripes);
@@ -504,6 +503,32 @@ class ORCFileReader::Impl {
@@ -479,6 +478,31 @@ class ORCFileReader::Impl {
 return Status::OK();
 }

+ Result<std::shared_ptr<RecordBatchReader>> NextStripeReader(
+ int64_t batch_size, const std::vector<std::string>& include_names) {
+ if (current_row_ >= NumberOfRows()) {
+ return nullptr;
+ }
+
+ liborc::RowReaderOptions opts;
+ liborc::RowReaderOptions opts = default_row_reader_options();
+ if (!include_names.empty()) {
+ RETURN_NOT_OK(SelectNames(&opts, include_names));
+ }
+ StripeInformation stripe_info({0, 0, 0, 0});
+ RETURN_NOT_OK(SelectStripeWithRowNumber(&opts, current_row_, &stripe_info));
+ std::shared_ptr<Schema> schema;
+ RETURN_NOT_OK(ReadSchema(opts, &schema));
+ ARROW_ASSIGN_OR_RAISE(auto schema, ReadSchema(opts));
+ std::unique_ptr<liborc::RowReader> row_reader;
+
+ ORC_BEGIN_CATCH_NOT_OK
+ row_reader = reader_->createRowReader(opts);
+ row_reader->seekToRow(current_row_);
+ current_row_ = stripe_info.first_row_of_stripe + stripe_info.num_rows;
+ current_row_ = stripe_info.first_row_id + stripe_info.num_rows;
+ ORC_END_CATCH_NOT_OK
+
+ return std::make_shared<OrcStripeReader>(std::move(row_reader), schema, batch_size,
+ pool_);
+ }
+
Status NextStripeReader(int64_t batch_size, std::shared_ptr<RecordBatchReader>* out) {
return NextStripeReader(batch_size, {}, out);
}
@@ -531,6 +556,8 @@ Result<std::unique_ptr<ORCFileReader>> ORCFileReader::Open(
Result<std::shared_ptr<RecordBatchReader>> NextStripeReader(
 int64_t batch_size, const std::vector<int>& include_indices) {
 if (current_row_ >= NumberOfRows()) {
@@ -544,6 +568,8 @@ Result<std::unique_ptr<ORCFileReader>> ORCFileReader::Open(
 return std::move(result);
 }

+liborc::Reader* ORCFileReader::GetRawORCReader() { return impl_->GetRawORCReader(); }
+
 Result<std::shared_ptr<const KeyValueMetadata>> ORCFileReader::ReadMetadata() {
 return impl_->ReadMetadata();
 }
@@ -653,6 +680,11 @@ Result<std::shared_ptr<RecordBatchReader>> ORCFileReader::NextStripeReader(
 return reader;
@@ -605,6 +631,11 @@ Result<std::shared_ptr<RecordBatchReader>> ORCFileReader::NextStripeReader(
 return impl_->NextStripeReader(batch_size, include_indices);
 }

+Result<std::shared_ptr<RecordBatchReader>> ORCFileReader::NextStripeReader(
+ int64_t batch_size, const std::vector<std::string>& include_names) {
+ return impl_->NextStripeReader(batch_size, include_names);
+}
+
 int64_t ORCFileReader::NumberOfStripes() { return impl_->NumberOfStripes(); }

 int64_t ORCFileReader::NumberOfRows() { return impl_->NumberOfRows(); }
diff --git a/cpp/src/arrow/adapters/orc/adapter.h b/cpp/src/arrow/adapters/orc/adapter.h
index 223efa515..04e6b0612 100644
index 013be7860..7fd06bcb8 100644
--- a/cpp/src/arrow/adapters/orc/adapter.h
+++ b/cpp/src/arrow/adapters/orc/adapter.h
@@ -30,6 +30,10 @@
Expand All@@ -87,23 +86,23 @@ index 223efa515..04e6b0612 100644
+
+// alias to not interfere with nested orc namespace
+namespace liborc = orc;

 namespace arrow {
 namespace adapters {
@@ -51,6 +55,9 @@ class ARROW_EXPORT ORCFileReader {
 static Status Open(const std::shared_ptr<io::RandomAccessFile>& file, MemoryPool* pool,
 std::unique_ptr<ORCFileReader>* reader);
@@ -53,6 +57,9 @@ class ARROW_EXPORT ORCFileReader {
public:
~ORCFileReader();

+ /// \brief Get ORC reader from inside.
+ liborc::Reader* GetRawORCReader();
+
 /// \brief Creates a new ORC reader
 ///
 /// \param[in] file the data source
@@ -240,6 +247,19 @@ class ARROW_EXPORT ORCFileReader {
 Result<std::shared_ptr<RecordBatchReader>> NextStripeReader(
 int64_t batch_size, const std::vector<int>& include_indices);
@@ -174,6 +181,19 @@ class ARROW_EXPORT ORCFileReader {
 Result<std::shared_ptr<RecordBatchReader>> GetRecordBatchReader(
 int64_t batch_size, const std::vector<std::string>& include_names);

+ /// \brief Get a stripe level record batch iterator with specified row count
+ /// in each record batch. NextStripeReader serves as a fine grain
+ /// alternative to ReadStripe which may cause OOM issue by loading
Expand All@@ -119,4 +118,3 @@ index 223efa515..04e6b0612 100644
+
 /// \brief The number of stripes in the file
 int64_t NumberOfStripes();


41 changes: 28 additions & 13 deletions
thirdparty/vars.sh

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -238,11 +238,24 @@ FLATBUFFERS_NAME=flatbuffers-2.0.0.tar.gz
FLATBUFFERS_SOURCE=flatbuffers-2.0.0
FLATBUFFERS_MD5SUM="a27992324c3cbf86dd888268a23d17bd"

# c-ares
CARES_DOWNLOAD="https://github.com/c-ares/c-ares/releases/download/cares-1_19_1/c-ares-1.19.1.tar.gz"
CARES_NAME="c-ares-1.19.1.tar.gz"
CARES_SOURCE=c-ares-1.19.1
CARES_MD5SUM="dafc5825a92dc907e144570e4e75a908"

# grpc
# grpc v1.55 and above require protobuf >= 22
GRPC_DOWNLOAD="https://github.com/grpc/grpc/archive/refs/tags/v1.54.3.tar.gz"
GRPC_NAME="grpc-v1.54.3.tar.gz"
GRPC_SOURCE=grpc-1.54.3
GRPC_MD5SUM="af00a2edeae0f02bb25917cc3473b7de"

# arrow
ARROW_DOWNLOAD="https://archive.apache.org/dist/arrow/arrow-7.0.0/apache-arrow-7.0.0.tar.gz"
ARROW_NAME="apache-arrow-7.0.0.tar.gz"
ARROW_SOURCE="apache-arrow-7.0.0"
ARROW_MD5SUM="316ade159901646849b3b4760fa52816"
ARROW_DOWNLOAD="https://github.com/apache/arrow/archive/refs/tags/apache-arrow-13.0.0.tar.gz"
ARROW_NAME="apache-arrow-13.0.0.tar.gz"
ARROW_SOURCE="arrow-apache-arrow-13.0.0"
ARROW_MD5SUM="8ec1ec6a119514bcaea1cf7aabc9df1f"

# Abseil
ABSEIL_DOWNLOAD="https://github.com/abseil/abseil-cpp/archive/refs/tags/20220623.1.tar.gz"
Expand DownExpand Up@@ -287,10 +300,10 @@ ORC_SOURCE=orc-1.7.2
ORC_MD5SUM="6cab37935eacdec7d078d327746a8578"

# jemalloc for arrow
JEMALLOC_ARROW_DOWNLOAD="https://github.com/jemalloc/jemalloc/releases/download/5.2.1/jemalloc-5.2.1.tar.bz2"
JEMALLOC_ARROW_NAME="jemalloc-5.2.1.tar.bz2"
JEMALLOC_ARROW_SOURCE="jemalloc-5.2.1"
JEMALLOC_ARROW_MD5SUM="3d41fbf006e6ebffd489bdb304d009ae"
JEMALLOC_ARROW_DOWNLOAD="https://github.com/jemalloc/jemalloc/releases/download/5.3.0/jemalloc-5.3.0.tar.bz2"
JEMALLOC_ARROW_NAME="jemalloc-5.3.0.tar.bz2"
JEMALLOC_ARROW_SOURCE="jemalloc-5.3.0"
JEMALLOC_ARROW_MD5SUM="09a8328574dab22a7df848eae6dbbf53"

# jemalloc for doris
JEMALLOC_DORIS_DOWNLOAD="https://github.com/jemalloc/jemalloc/releases/download/5.3.0/jemalloc-5.3.0.tar.bz2"
Expand DownExpand Up@@ -398,11 +411,11 @@ BENCHMARK_SOURCE=benchmark-1.8.0
BENCHMARK_MD5SUM="8ddf8571d3f6198d37852bcbd964f817"

# xsimd
# for arrow-7.0.0, if arrow upgrade, this version may also need to be changed
XSIMD_DOWNLOAD="https://github.com/xtensor-stack/xsimd/archive/aeec9c872c8b475dedd7781336710f2dd2666cb2.tar.gz"
XSIMD_NAME=xsimd-aeec9c872c8b475dedd7781336710f2dd2666cb2.tar.gz
XSIMD_SOURCE=xsimd-aeec9c872c8b475dedd7781336710f2dd2666cb2
XSIMD_MD5SUM="d024855f71c0a2837a6918c0f8f66245"
# for arrow-13.0.0, if arrow upgrade, this version may also need to be changed
XSIMD_DOWNLOAD="https://github.com/xtensor-stack/xsimd/archive/refs/tags/9.0.1.tar.gz"
XSIMD_NAME="xsimd-9.0.1.tar.gz"
XSIMD_SOURCE=xsimd-9.0.1
XSIMD_MD5SUM="59f38fe3364acd7ed137771258812d6c"

# simdjson
SIMDJSON_DOWNLOAD="https://github.com/simdjson/simdjson/archive/refs/tags/v3.0.1.tar.gz"
Expand DownExpand Up@@ -505,6 +518,8 @@ export TP_ARCHIVES=(
 'CYRUS_SASL'
 'LIBRDKAFKA'
 'FLATBUFFERS'
 'CARES'
 'GRPC'
 'ARROW'
 'BROTLI'
 'ZSTD'
Expand Down

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Highlight search terms from Google/DuckDuckGo/Bing referrer\n(function() {\n var ref = document.referrer;\n var terms = [];\n \n if (ref.includes('google.com') || ref.includes('duckduckgo.com') || ref.includes('bing.com')) {\n var url = new URL(ref);\n var q = url.searchParams.get('q') || url.searchParams.get('p');\n if (q) {\n terms = q.split(/\\s+/).filter(function(t) { return t.length > 2; });\n }\n }\n \n if (terms.length === 0) return;\n \n var style = document.createElement('style');\n style.textContent = '.userscript-highlight { background: #fbbf24; color: #1a1a2e; padding: 1px 3px; border-radius: 2px; }';\n document.head.appendChild(style);\n \n function highlight(node) {\n if (node.nodeType === 3) { // text node\n var text = node.textContent;\n var found = false;\n terms.forEach(function(term) {\n var regex = new RegExp('(' + term.replace(/[.*+?^${}()|[\\]\\\\]/g, '\\\\') + ')', 'gi');\n if (regex.test(text)) {\n found = true;\n var frag = document.createDocumentFragment();\n var parts = text.split(regex);\n parts.forEach(function(part, i) {\n if (i % 2 === 0) {\n frag.appendChild(document.createTextNode(part));\n } else {\n var span = document.createElement('span');\n span.className = 'userscript-highlight';\n span.textContent = part;\n frag.appendChild(span);\n }\n });\n node.parentNode.replaceChild(frag, node);\n }\n });\n } else if (node.nodeType === 1 && node.childNodes) { // element\n var skipTags = ['SCRIPT', 'STYLE', 'NOSCRIPT', 'TEXTAREA', 'INPUT', 'SELECT'];\n if (!skipTags.includes(node.tagName)) {\n Array.from(node.childNodes).forEach(highlight);\n }\n }\n }\n \n highlight(document.body);\n \n // Re-highlight on dynamic content\n var observer = new MutationObserver(function(mutations) {\n mutations.forEach(function(m) {\n m.addedNodes.forEach(function(node) {\n if (node.nodeType === 1 || node.nodeType === 3) highlight(node);\n });\n });\n });\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Highlight Search Terms");
}
} catch(__e) { console.warn('[Userscript:Highlight Search Terms]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
[Fix](Outfile) upgrade apache-arrow version to 13.0.0#35142

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Closed

BePPPower
wants to merge
3
commits into

from

 Closed


[Fix](Outfile) upgrade apache-arrow version to 13.0.0
#35142
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


3 commits


Select commit
Hold shift + click to select a range
466bddc

upgrade


BePPPower May 21, 2024
1d2137c

fix


BePPPower May 22, 2024
ff07338

fix 2


BePPPower May 22, 2024

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


7 changes: 7 additions & 0 deletions
thirdparty/CHANGELOG.md

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -170,6 +170,13 @@ Now there will be 2 set of libhdfs, one is without kerberos, the other is with k

## v20211215

## v20240521
- Modified: arrow 7.0.0 -> 13.0.0
- Modified: jemalloc for arrow 5.2.1 -> 5.3.0
- Modified: xsimd 7.0.0 -> 13.0.0
- Added: c-ares -> 1.19.1
- Added: grpc -> 1.54.3

### Changes

- Added: cyrus-sasl
Expand Down

72 changes: 71 additions & 1 deletion
thirdparty/build-thirdparty.sh

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -958,6 +958,8 @@ build_arrow() {
 export ARROW_ZLIB_URL="${TP_SOURCE_DIR}/${ZLIB_NAME}"
 export ARROW_XSIMD_URL="${TP_SOURCE_DIR}/${XSIMD_NAME}"
 export ARROW_ORC_URL="${TP_SOURCE_DIR}/${ORC_NAME}"
 export ARROW_GRPC_URL="${TP_SOURCE_DIR}/${GRPC_NAME}"
 export ARROW_PROTOBUF_URL="${TP_SOURCE_DIR}/${PROTOBUF_NAME}"

 if [[ "${KERNEL}" != 'Darwin' ]]; then
 ldflags="-L${TP_LIB_DIR} -static-libstdc++ -static-libgcc"
Expand All@@ -973,22 +975,38 @@ build_arrow() {
 -DCMAKE_INSTALL_PREFIX="${TP_INSTALL_DIR}" \
 -DCMAKE_INSTALL_LIBDIR=lib64 \
 -DARROW_BOOST_USE_SHARED=OFF \
 -DARROW_WITH_GRPC=ON \
 -DgRPC_SOURCE=SYSTEM \
 -DgRPC_ROOT="${TP_INSTALL_DIR}" \
 -DARROW_WITH_PROTOBUF=ON \
 -DProtobuf_SOURCE=SYSTEM \
 -DProtobuf_LIB="${TP_INSTALL_DIR}/lib/libprotoc.a" -DProtobuf_INCLUDE_DIR="${TP_INSTALL_DIR}/include" \
 -DARROW_FLIGHT=ON \
 -DARROW_FLIGHT_SQL=ON \
 -DBoost_USE_STATIC_RUNTIME=ON \
 -DARROW_GFLAGS_USE_SHARED=OFF \
 -Dgflags_ROOT="${TP_INSTALL_DIR}" \
 -DGLOG_ROOT="${TP_INSTALL_DIR}" \
 -DRE2_ROOT="${TP_INSTALL_DIR}" \
 -DZLIB_SOURCE=SYSTEM \
 -DZLIB_LIBRARY="${TP_INSTALL_DIR}/lib/libz.a" -DZLIB_INCLUDE_DIR="${TP_INSTALL_DIR}/include" \
 -DRapidJSON_SOURCE=SYSTEM \
 -DRapidJSON_ROOT="${TP_INSTALL_DIR}" \
 -DORC_ROOT="${TP_INSTALL_DIR}" \
 -Dxsimd_SOURCE=BUNDLED \
 -DBrotli_SOURCE=BUNDLED \
 -DARROW_LZ4_USE_SHARED=OFF \
 -DLZ4_LIB="${TP_INSTALL_DIR}/lib/liblz4.a" -DLZ4_INCLUDE_DIR="${TP_INSTALL_DIR}/include/lz4" \
 -DLz4_SOURCE=SYSTEM \
 -DARROW_ZSTD_USE_SHARED=OFF \
 -DZSTD_LIB="${TP_INSTALL_DIR}/lib/libzstd.a" -DZSTD_INCLUDE_DIR="${TP_INSTALL_DIR}/include" \
 -Dzstd_SOURCE=SYSTEM \
 -DSnappy_LIB="${TP_INSTALL_DIR}/lib/libsnappy.a" -DSnappy_INCLUDE_DIR="${TP_INSTALL_DIR}/include" \
 -DSnappy_SOURCE=SYSTEM \
 -DBOOST_ROOT="${TP_INSTALL_DIR}" --no-warn-unused-cli \
 -Djemalloc_SOURCE=BUNDLED \
 -DARROW_THRIFT_USE_SHARED=OFF \
 -DThrift_SOURCE=SYSTEM \
 -DThrift_ROOT="${TP_INSTALL_DIR}" ..

 "${BUILD_SYSTEM}" -j "${PARALLEL}"
Expand DownExpand Up@@ -1662,6 +1680,56 @@ build_libdeflate() {
 "${BUILD_SYSTEM}" install
}

# c-ares
build_cares() {
 check_if_source_exist "${CARES_SOURCE}"
 cd "${TP_SOURCE_DIR}/${CARES_SOURCE}"

 mkdir -p build
 cd build
 cmake -DCMAKE_BUILD_TYPE=Release \
 -DCARES_STATIC=ON \
 -DCARES_SHARED=OFF \
 -DCARES_STATIC_PIC=ON \
 -DCMAKE_INSTALL_PREFIX="${TP_INSTALL_DIR}" ..
 make
 make install
}

# grpc
build_grpc() {
 check_if_source_exist "${GRPC_SOURCE}"
 cd "${TP_SOURCE_DIR}/${GRPC_SOURCE}"

 mkdir -p cmake/build
 cd cmake/build

 cmake -DgRPC_INSTALL=ON \
 -DgRPC_BUILD_TESTS=OFF \
 -DCMAKE_BUILD_TYPE=Release \
 -DCMAKE_INSTALL_PREFIX="${TP_INSTALL_DIR}" \
 -DgRPC_CARES_PROVIDER=package \
 -Dc-ares_DIR="${TP_INSTALL_DIR}" \
 -DgRPC_ABSL_PROVIDER=package \
 -Dabsl_DIR="${TP_INSTALL_DIR}" \
 -DgRPC_PROTOBUF_PROVIDER=package \
 -DProtobuf_DIR="${TP_INSTALL_DIR}" \
 -DgRPC_RE2_PROVIDER=package \
 -Dre2_DIR:STRING="${TP_INSTALL_DIR}" \
 -DgRPC_SSL_PROVIDER=package \
 -DOPENSSL_ROOT_DIR="${TP_INSTALL_DIR}" \
 -DgRPC_ZLIB_PROVIDER=package \
 -DZLIB_ROOT="${TP_INSTALL_DIR}" \
 -DCMAKE_POSITION_INDEPENDENT_CODE=ON \
 ../..

 make -j "${PARALLEL}"
 make install

 # for grpc > v1.55, cmake 2.22 does not support find_dependency, delete this line after cmake version upgrade.
 # sed -i 's/find_dependency/find_package/g' "${TP_INSTALL_DIR}"/lib64/cmake/grpc/gRPCConfig.cmake
}

if [[ "${#packages[@]}" -eq 0 ]]; then
 packages=(
 libunixodbc
Expand All@@ -1673,9 +1741,9 @@ if [[ "${#packages[@]}" -eq 0 ]]; then
 lzo2
 zstd
 boost # must before thrift
 protobuf
 gflags
 gtest
 protobuf # after gtest
 glog
 rapidjson
 snappy
Expand All@@ -1693,6 +1761,8 @@ if [[ "${#packages[@]}" -eq 0 ]]; then
 librdkafka
 flatbuffers
 orc
 cares
 grpc # after cares, protobuf
 arrow
 abseil
 s2
Expand Down

4 changes: 2 additions & 2 deletions
thirdparty/download-thirdparty.sh

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -323,10 +323,10 @@ fi
echo "Finished patching ${OPENTELEMETRY_SOURCE}"

# arrow patch is used to get the raw orc reader for filter prune.
if [[ "${ARROW_SOURCE}" == "apache-arrow-7.0.0" ]]; then
if [[ "${ARROW_SOURCE}" == "arrow-apache-arrow-13.0.0" ]]; then
 cd "${TP_SOURCE_DIR}/${ARROW_SOURCE}"
 if [[ ! -f "${PATCHED_MARK}" ]]; then
 patch -p1 <"${TP_PATCH_DIR}/apache-arrow-7.0.0.patch"
 patch -p1 <"${TP_PATCH_DIR}/apache-arrow-13.0.0.patch"
 touch "${PATCHED_MARK}"
 fi
 cd -
Expand Down

58 changes: 28 additions & 30 deletions
thirdparty/patches/apache-arrow-7.0.0.patch → thirdparty/patches/apache-arrow-13.0.0.patch

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -1,82 +1,81 @@
diff --git a/cpp/src/arrow/adapters/orc/adapter.cc b/cpp/src/arrow/adapters/orc/adapter.cc
index 03243e751..1eb9b2c81 100644
index 2466e7433..46b4402d4 100644
--- a/cpp/src/arrow/adapters/orc/adapter.cc
+++ b/cpp/src/arrow/adapters/orc/adapter.cc
@@ -47,9 +47,6 @@
 #include "arrow/util/visibility.h"
 #include "orc/Exceptions.hh"

-// alias to not interfere with nested orc namespace
-namespace liborc = orc;
-
 #define ORC_THROW_NOT_OK(s) \
 do { \
 Status _s = (s); \
@@ -198,6 +195,8 @@ class ORCFileReader::Impl {
@@ -202,6 +199,8 @@ class ORCFileReader::Impl {
 return Init();
 }

+ virtual liborc::Reader* GetRawORCReader() { return reader_.get(); }
+
 Status Init() {
 int64_t nstripes = reader_->getNumberOfStripes();
 stripes_.resize(nstripes);
@@ -504,6 +503,32 @@ class ORCFileReader::Impl {
@@ -479,6 +478,31 @@ class ORCFileReader::Impl {
 return Status::OK();
 }

+ Result<std::shared_ptr<RecordBatchReader>> NextStripeReader(
+ int64_t batch_size, const std::vector<std::string>& include_names) {
+ if (current_row_ >= NumberOfRows()) {
+ return nullptr;
+ }
+
+ liborc::RowReaderOptions opts;
+ liborc::RowReaderOptions opts = default_row_reader_options();
+ if (!include_names.empty()) {
+ RETURN_NOT_OK(SelectNames(&opts, include_names));
+ }
+ StripeInformation stripe_info({0, 0, 0, 0});
+ RETURN_NOT_OK(SelectStripeWithRowNumber(&opts, current_row_, &stripe_info));
+ std::shared_ptr<Schema> schema;
+ RETURN_NOT_OK(ReadSchema(opts, &schema));
+ ARROW_ASSIGN_OR_RAISE(auto schema, ReadSchema(opts));
+ std::unique_ptr<liborc::RowReader> row_reader;
+
+ ORC_BEGIN_CATCH_NOT_OK
+ row_reader = reader_->createRowReader(opts);
+ row_reader->seekToRow(current_row_);
+ current_row_ = stripe_info.first_row_of_stripe + stripe_info.num_rows;
+ current_row_ = stripe_info.first_row_id + stripe_info.num_rows;
+ ORC_END_CATCH_NOT_OK
+
+ return std::make_shared<OrcStripeReader>(std::move(row_reader), schema, batch_size,
+ pool_);
+ }
+
Status NextStripeReader(int64_t batch_size, std::shared_ptr<RecordBatchReader>* out) {
return NextStripeReader(batch_size, {}, out);
}
@@ -531,6 +556,8 @@ Result<std::unique_ptr<ORCFileReader>> ORCFileReader::Open(
Result<std::shared_ptr<RecordBatchReader>> NextStripeReader(
 int64_t batch_size, const std::vector<int>& include_indices) {
 if (current_row_ >= NumberOfRows()) {
@@ -544,6 +568,8 @@ Result<std::unique_ptr<ORCFileReader>> ORCFileReader::Open(
 return std::move(result);
 }

+liborc::Reader* ORCFileReader::GetRawORCReader() { return impl_->GetRawORCReader(); }
+
 Result<std::shared_ptr<const KeyValueMetadata>> ORCFileReader::ReadMetadata() {
 return impl_->ReadMetadata();
 }
@@ -653,6 +680,11 @@ Result<std::shared_ptr<RecordBatchReader>> ORCFileReader::NextStripeReader(
 return reader;
@@ -605,6 +631,11 @@ Result<std::shared_ptr<RecordBatchReader>> ORCFileReader::NextStripeReader(
 return impl_->NextStripeReader(batch_size, include_indices);
 }

+Result<std::shared_ptr<RecordBatchReader>> ORCFileReader::NextStripeReader(
+ int64_t batch_size, const std::vector<std::string>& include_names) {
+ return impl_->NextStripeReader(batch_size, include_names);
+}
+
 int64_t ORCFileReader::NumberOfStripes() { return impl_->NumberOfStripes(); }

 int64_t ORCFileReader::NumberOfRows() { return impl_->NumberOfRows(); }
diff --git a/cpp/src/arrow/adapters/orc/adapter.h b/cpp/src/arrow/adapters/orc/adapter.h
index 223efa515..04e6b0612 100644
index 013be7860..7fd06bcb8 100644
--- a/cpp/src/arrow/adapters/orc/adapter.h
+++ b/cpp/src/arrow/adapters/orc/adapter.h
@@ -30,6 +30,10 @@
Expand All@@ -87,23 +86,23 @@ index 223efa515..04e6b0612 100644
+
+// alias to not interfere with nested orc namespace
+namespace liborc = orc;

 namespace arrow {
 namespace adapters {
@@ -51,6 +55,9 @@ class ARROW_EXPORT ORCFileReader {
 static Status Open(const std::shared_ptr<io::RandomAccessFile>& file, MemoryPool* pool,
 std::unique_ptr<ORCFileReader>* reader);
@@ -53,6 +57,9 @@ class ARROW_EXPORT ORCFileReader {
public:
~ORCFileReader();

+ /// \brief Get ORC reader from inside.
+ liborc::Reader* GetRawORCReader();
+
 /// \brief Creates a new ORC reader
 ///
 /// \param[in] file the data source
@@ -240,6 +247,19 @@ class ARROW_EXPORT ORCFileReader {
 Result<std::shared_ptr<RecordBatchReader>> NextStripeReader(
 int64_t batch_size, const std::vector<int>& include_indices);
@@ -174,6 +181,19 @@ class ARROW_EXPORT ORCFileReader {
 Result<std::shared_ptr<RecordBatchReader>> GetRecordBatchReader(
 int64_t batch_size, const std::vector<std::string>& include_names);

+ /// \brief Get a stripe level record batch iterator with specified row count
+ /// in each record batch. NextStripeReader serves as a fine grain
+ /// alternative to ReadStripe which may cause OOM issue by loading
Expand All@@ -119,4 +118,3 @@ index 223efa515..04e6b0612 100644
+
 /// \brief The number of stripes in the file
 int64_t NumberOfStripes();


41 changes: 28 additions & 13 deletions
thirdparty/vars.sh

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -238,11 +238,24 @@ FLATBUFFERS_NAME=flatbuffers-2.0.0.tar.gz
FLATBUFFERS_SOURCE=flatbuffers-2.0.0
FLATBUFFERS_MD5SUM="a27992324c3cbf86dd888268a23d17bd"

# c-ares
CARES_DOWNLOAD="https://github.com/c-ares/c-ares/releases/download/cares-1_19_1/c-ares-1.19.1.tar.gz"
CARES_NAME="c-ares-1.19.1.tar.gz"
CARES_SOURCE=c-ares-1.19.1
CARES_MD5SUM="dafc5825a92dc907e144570e4e75a908"

# grpc
# grpc v1.55 and above require protobuf >= 22
GRPC_DOWNLOAD="https://github.com/grpc/grpc/archive/refs/tags/v1.54.3.tar.gz"
GRPC_NAME="grpc-v1.54.3.tar.gz"
GRPC_SOURCE=grpc-1.54.3
GRPC_MD5SUM="af00a2edeae0f02bb25917cc3473b7de"

# arrow
ARROW_DOWNLOAD="https://archive.apache.org/dist/arrow/arrow-7.0.0/apache-arrow-7.0.0.tar.gz"
ARROW_NAME="apache-arrow-7.0.0.tar.gz"
ARROW_SOURCE="apache-arrow-7.0.0"
ARROW_MD5SUM="316ade159901646849b3b4760fa52816"
ARROW_DOWNLOAD="https://github.com/apache/arrow/archive/refs/tags/apache-arrow-13.0.0.tar.gz"
ARROW_NAME="apache-arrow-13.0.0.tar.gz"
ARROW_SOURCE="arrow-apache-arrow-13.0.0"
ARROW_MD5SUM="8ec1ec6a119514bcaea1cf7aabc9df1f"

# Abseil
ABSEIL_DOWNLOAD="https://github.com/abseil/abseil-cpp/archive/refs/tags/20220623.1.tar.gz"
Expand DownExpand Up@@ -287,10 +300,10 @@ ORC_SOURCE=orc-1.7.2
ORC_MD5SUM="6cab37935eacdec7d078d327746a8578"

# jemalloc for arrow
JEMALLOC_ARROW_DOWNLOAD="https://github.com/jemalloc/jemalloc/releases/download/5.2.1/jemalloc-5.2.1.tar.bz2"
JEMALLOC_ARROW_NAME="jemalloc-5.2.1.tar.bz2"
JEMALLOC_ARROW_SOURCE="jemalloc-5.2.1"
JEMALLOC_ARROW_MD5SUM="3d41fbf006e6ebffd489bdb304d009ae"
JEMALLOC_ARROW_DOWNLOAD="https://github.com/jemalloc/jemalloc/releases/download/5.3.0/jemalloc-5.3.0.tar.bz2"
JEMALLOC_ARROW_NAME="jemalloc-5.3.0.tar.bz2"
JEMALLOC_ARROW_SOURCE="jemalloc-5.3.0"
JEMALLOC_ARROW_MD5SUM="09a8328574dab22a7df848eae6dbbf53"

# jemalloc for doris
JEMALLOC_DORIS_DOWNLOAD="https://github.com/jemalloc/jemalloc/releases/download/5.3.0/jemalloc-5.3.0.tar.bz2"
Expand DownExpand Up@@ -398,11 +411,11 @@ BENCHMARK_SOURCE=benchmark-1.8.0
BENCHMARK_MD5SUM="8ddf8571d3f6198d37852bcbd964f817"

# xsimd
# for arrow-7.0.0, if arrow upgrade, this version may also need to be changed
XSIMD_DOWNLOAD="https://github.com/xtensor-stack/xsimd/archive/aeec9c872c8b475dedd7781336710f2dd2666cb2.tar.gz"
XSIMD_NAME=xsimd-aeec9c872c8b475dedd7781336710f2dd2666cb2.tar.gz
XSIMD_SOURCE=xsimd-aeec9c872c8b475dedd7781336710f2dd2666cb2
XSIMD_MD5SUM="d024855f71c0a2837a6918c0f8f66245"
# for arrow-13.0.0, if arrow upgrade, this version may also need to be changed
XSIMD_DOWNLOAD="https://github.com/xtensor-stack/xsimd/archive/refs/tags/9.0.1.tar.gz"
XSIMD_NAME="xsimd-9.0.1.tar.gz"
XSIMD_SOURCE=xsimd-9.0.1
XSIMD_MD5SUM="59f38fe3364acd7ed137771258812d6c"

# simdjson
SIMDJSON_DOWNLOAD="https://github.com/simdjson/simdjson/archive/refs/tags/v3.0.1.tar.gz"
Expand DownExpand Up@@ -505,6 +518,8 @@ export TP_ARCHIVES=(
 'CYRUS_SASL'
 'LIBRDKAFKA'
 'FLATBUFFERS'
 'CARES'
 'GRPC'
 'ARROW'
 'BROTLI'
 'ZSTD'
Expand Down

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Strip utm_, fbclid, gclid, etc. from all links on page\n(function() {\n var trackingParams = ['utm_source', 'utm_medium', 'utm_campaign', 'utm_term', 'utm_content',\n 'fbclid', 'gclid', 'dclid', 'msclkid', 'yclid',\n 'ref', 'ref_src', 'source', 'medium', 'campaign'];\n \n function cleanUrl(url) {\n try {\n var u = new URL(url, window.location.origin);\n var changed = false;\n trackingParams.forEach(function(p) {\n if (u.searchParams.has(p)) {\n u.searchParams.delete(p);\n changed = true;\n }\n });\n return changed ? u.toString() : url;\n } catch (e) {\n return url;\n }\n }\n \n function cleanLinks() {\n document.querySelectorAll('a[href]').forEach(function(a) {\n var clean = cleanUrl(a.href);\n if (clean !== a.href) a.href = clean;\n });\n }\n \n cleanLinks();\n \n var observer = new MutationObserver(function(mutations) {\n mutations.forEach(function(m) {\n m.addedNodes.forEach(function(node) {\n if (node.nodeType === 1) {\n if (node.tagName === 'A') cleanLinks();\n node.querySelectorAll('a[href]').forEach(function(a) {\n var clean = cleanUrl(a.href);\n if (clean !== a.href) a.href = clean;\n });\n }\n });\n });\n });\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Remove Tracking Parameters from Links");
}
} catch(__e) { console.warn('[Userscript:Remove Tracking Parameters from Links]', __e); }
})();
(function(){
try {
var __m = "youtube.com";
var __re = new RegExp('^' + "youtube\\.com" + '

Skip to content
[Fix](Outfile) upgrade apache-arrow version to 13.0.0#35142

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Closed

BePPPower
wants to merge
3
commits into

from

 Closed


[Fix](Outfile) upgrade apache-arrow version to 13.0.0
#35142
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


3 commits


Select commit
Hold shift + click to select a range
466bddc

upgrade


BePPPower May 21, 2024
1d2137c

fix


BePPPower May 22, 2024
ff07338

fix 2


BePPPower May 22, 2024

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


7 changes: 7 additions & 0 deletions
thirdparty/CHANGELOG.md

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -170,6 +170,13 @@ Now there will be 2 set of libhdfs, one is without kerberos, the other is with k

## v20211215

## v20240521
- Modified: arrow 7.0.0 -> 13.0.0
- Modified: jemalloc for arrow 5.2.1 -> 5.3.0
- Modified: xsimd 7.0.0 -> 13.0.0
- Added: c-ares -> 1.19.1
- Added: grpc -> 1.54.3

### Changes

- Added: cyrus-sasl
Expand Down

72 changes: 71 additions & 1 deletion
thirdparty/build-thirdparty.sh

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -958,6 +958,8 @@ build_arrow() {
 export ARROW_ZLIB_URL="${TP_SOURCE_DIR}/${ZLIB_NAME}"
 export ARROW_XSIMD_URL="${TP_SOURCE_DIR}/${XSIMD_NAME}"
 export ARROW_ORC_URL="${TP_SOURCE_DIR}/${ORC_NAME}"
 export ARROW_GRPC_URL="${TP_SOURCE_DIR}/${GRPC_NAME}"
 export ARROW_PROTOBUF_URL="${TP_SOURCE_DIR}/${PROTOBUF_NAME}"

 if [[ "${KERNEL}" != 'Darwin' ]]; then
 ldflags="-L${TP_LIB_DIR} -static-libstdc++ -static-libgcc"
Expand All@@ -973,22 +975,38 @@ build_arrow() {
 -DCMAKE_INSTALL_PREFIX="${TP_INSTALL_DIR}" \
 -DCMAKE_INSTALL_LIBDIR=lib64 \
 -DARROW_BOOST_USE_SHARED=OFF \
 -DARROW_WITH_GRPC=ON \
 -DgRPC_SOURCE=SYSTEM \
 -DgRPC_ROOT="${TP_INSTALL_DIR}" \
 -DARROW_WITH_PROTOBUF=ON \
 -DProtobuf_SOURCE=SYSTEM \
 -DProtobuf_LIB="${TP_INSTALL_DIR}/lib/libprotoc.a" -DProtobuf_INCLUDE_DIR="${TP_INSTALL_DIR}/include" \
 -DARROW_FLIGHT=ON \
 -DARROW_FLIGHT_SQL=ON \
 -DBoost_USE_STATIC_RUNTIME=ON \
 -DARROW_GFLAGS_USE_SHARED=OFF \
 -Dgflags_ROOT="${TP_INSTALL_DIR}" \
 -DGLOG_ROOT="${TP_INSTALL_DIR}" \
 -DRE2_ROOT="${TP_INSTALL_DIR}" \
 -DZLIB_SOURCE=SYSTEM \
 -DZLIB_LIBRARY="${TP_INSTALL_DIR}/lib/libz.a" -DZLIB_INCLUDE_DIR="${TP_INSTALL_DIR}/include" \
 -DRapidJSON_SOURCE=SYSTEM \
 -DRapidJSON_ROOT="${TP_INSTALL_DIR}" \
 -DORC_ROOT="${TP_INSTALL_DIR}" \
 -Dxsimd_SOURCE=BUNDLED \
 -DBrotli_SOURCE=BUNDLED \
 -DARROW_LZ4_USE_SHARED=OFF \
 -DLZ4_LIB="${TP_INSTALL_DIR}/lib/liblz4.a" -DLZ4_INCLUDE_DIR="${TP_INSTALL_DIR}/include/lz4" \
 -DLz4_SOURCE=SYSTEM \
 -DARROW_ZSTD_USE_SHARED=OFF \
 -DZSTD_LIB="${TP_INSTALL_DIR}/lib/libzstd.a" -DZSTD_INCLUDE_DIR="${TP_INSTALL_DIR}/include" \
 -Dzstd_SOURCE=SYSTEM \
 -DSnappy_LIB="${TP_INSTALL_DIR}/lib/libsnappy.a" -DSnappy_INCLUDE_DIR="${TP_INSTALL_DIR}/include" \
 -DSnappy_SOURCE=SYSTEM \
 -DBOOST_ROOT="${TP_INSTALL_DIR}" --no-warn-unused-cli \
 -Djemalloc_SOURCE=BUNDLED \
 -DARROW_THRIFT_USE_SHARED=OFF \
 -DThrift_SOURCE=SYSTEM \
 -DThrift_ROOT="${TP_INSTALL_DIR}" ..

 "${BUILD_SYSTEM}" -j "${PARALLEL}"
Expand DownExpand Up@@ -1662,6 +1680,56 @@ build_libdeflate() {
 "${BUILD_SYSTEM}" install
}

# c-ares
build_cares() {
 check_if_source_exist "${CARES_SOURCE}"
 cd "${TP_SOURCE_DIR}/${CARES_SOURCE}"

 mkdir -p build
 cd build
 cmake -DCMAKE_BUILD_TYPE=Release \
 -DCARES_STATIC=ON \
 -DCARES_SHARED=OFF \
 -DCARES_STATIC_PIC=ON \
 -DCMAKE_INSTALL_PREFIX="${TP_INSTALL_DIR}" ..
 make
 make install
}

# grpc
build_grpc() {
 check_if_source_exist "${GRPC_SOURCE}"
 cd "${TP_SOURCE_DIR}/${GRPC_SOURCE}"

 mkdir -p cmake/build
 cd cmake/build

 cmake -DgRPC_INSTALL=ON \
 -DgRPC_BUILD_TESTS=OFF \
 -DCMAKE_BUILD_TYPE=Release \
 -DCMAKE_INSTALL_PREFIX="${TP_INSTALL_DIR}" \
 -DgRPC_CARES_PROVIDER=package \
 -Dc-ares_DIR="${TP_INSTALL_DIR}" \
 -DgRPC_ABSL_PROVIDER=package \
 -Dabsl_DIR="${TP_INSTALL_DIR}" \
 -DgRPC_PROTOBUF_PROVIDER=package \
 -DProtobuf_DIR="${TP_INSTALL_DIR}" \
 -DgRPC_RE2_PROVIDER=package \
 -Dre2_DIR:STRING="${TP_INSTALL_DIR}" \
 -DgRPC_SSL_PROVIDER=package \
 -DOPENSSL_ROOT_DIR="${TP_INSTALL_DIR}" \
 -DgRPC_ZLIB_PROVIDER=package \
 -DZLIB_ROOT="${TP_INSTALL_DIR}" \
 -DCMAKE_POSITION_INDEPENDENT_CODE=ON \
 ../..

 make -j "${PARALLEL}"
 make install

 # for grpc > v1.55, cmake 2.22 does not support find_dependency, delete this line after cmake version upgrade.
 # sed -i 's/find_dependency/find_package/g' "${TP_INSTALL_DIR}"/lib64/cmake/grpc/gRPCConfig.cmake
}

if [[ "${#packages[@]}" -eq 0 ]]; then
 packages=(
 libunixodbc
Expand All@@ -1673,9 +1741,9 @@ if [[ "${#packages[@]}" -eq 0 ]]; then
 lzo2
 zstd
 boost # must before thrift
 protobuf
 gflags
 gtest
 protobuf # after gtest
 glog
 rapidjson
 snappy
Expand All@@ -1693,6 +1761,8 @@ if [[ "${#packages[@]}" -eq 0 ]]; then
 librdkafka
 flatbuffers
 orc
 cares
 grpc # after cares, protobuf
 arrow
 abseil
 s2
Expand Down

4 changes: 2 additions & 2 deletions
thirdparty/download-thirdparty.sh

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -323,10 +323,10 @@ fi
echo "Finished patching ${OPENTELEMETRY_SOURCE}"

# arrow patch is used to get the raw orc reader for filter prune.
if [[ "${ARROW_SOURCE}" == "apache-arrow-7.0.0" ]]; then
if [[ "${ARROW_SOURCE}" == "arrow-apache-arrow-13.0.0" ]]; then
 cd "${TP_SOURCE_DIR}/${ARROW_SOURCE}"
 if [[ ! -f "${PATCHED_MARK}" ]]; then
 patch -p1 <"${TP_PATCH_DIR}/apache-arrow-7.0.0.patch"
 patch -p1 <"${TP_PATCH_DIR}/apache-arrow-13.0.0.patch"
 touch "${PATCHED_MARK}"
 fi
 cd -
Expand Down

58 changes: 28 additions & 30 deletions
thirdparty/patches/apache-arrow-7.0.0.patch → thirdparty/patches/apache-arrow-13.0.0.patch

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -1,82 +1,81 @@
diff --git a/cpp/src/arrow/adapters/orc/adapter.cc b/cpp/src/arrow/adapters/orc/adapter.cc
index 03243e751..1eb9b2c81 100644
index 2466e7433..46b4402d4 100644
--- a/cpp/src/arrow/adapters/orc/adapter.cc
+++ b/cpp/src/arrow/adapters/orc/adapter.cc
@@ -47,9 +47,6 @@
 #include "arrow/util/visibility.h"
 #include "orc/Exceptions.hh"

-// alias to not interfere with nested orc namespace
-namespace liborc = orc;
-
 #define ORC_THROW_NOT_OK(s) \
 do { \
 Status _s = (s); \
@@ -198,6 +195,8 @@ class ORCFileReader::Impl {
@@ -202,6 +199,8 @@ class ORCFileReader::Impl {
 return Init();
 }

+ virtual liborc::Reader* GetRawORCReader() { return reader_.get(); }
+
 Status Init() {
 int64_t nstripes = reader_->getNumberOfStripes();
 stripes_.resize(nstripes);
@@ -504,6 +503,32 @@ class ORCFileReader::Impl {
@@ -479,6 +478,31 @@ class ORCFileReader::Impl {
 return Status::OK();
 }

+ Result<std::shared_ptr<RecordBatchReader>> NextStripeReader(
+ int64_t batch_size, const std::vector<std::string>& include_names) {
+ if (current_row_ >= NumberOfRows()) {
+ return nullptr;
+ }
+
+ liborc::RowReaderOptions opts;
+ liborc::RowReaderOptions opts = default_row_reader_options();
+ if (!include_names.empty()) {
+ RETURN_NOT_OK(SelectNames(&opts, include_names));
+ }
+ StripeInformation stripe_info({0, 0, 0, 0});
+ RETURN_NOT_OK(SelectStripeWithRowNumber(&opts, current_row_, &stripe_info));
+ std::shared_ptr<Schema> schema;
+ RETURN_NOT_OK(ReadSchema(opts, &schema));
+ ARROW_ASSIGN_OR_RAISE(auto schema, ReadSchema(opts));
+ std::unique_ptr<liborc::RowReader> row_reader;
+
+ ORC_BEGIN_CATCH_NOT_OK
+ row_reader = reader_->createRowReader(opts);
+ row_reader->seekToRow(current_row_);
+ current_row_ = stripe_info.first_row_of_stripe + stripe_info.num_rows;
+ current_row_ = stripe_info.first_row_id + stripe_info.num_rows;
+ ORC_END_CATCH_NOT_OK
+
+ return std::make_shared<OrcStripeReader>(std::move(row_reader), schema, batch_size,
+ pool_);
+ }
+
Status NextStripeReader(int64_t batch_size, std::shared_ptr<RecordBatchReader>* out) {
return NextStripeReader(batch_size, {}, out);
}
@@ -531,6 +556,8 @@ Result<std::unique_ptr<ORCFileReader>> ORCFileReader::Open(
Result<std::shared_ptr<RecordBatchReader>> NextStripeReader(
 int64_t batch_size, const std::vector<int>& include_indices) {
 if (current_row_ >= NumberOfRows()) {
@@ -544,6 +568,8 @@ Result<std::unique_ptr<ORCFileReader>> ORCFileReader::Open(
 return std::move(result);
 }

+liborc::Reader* ORCFileReader::GetRawORCReader() { return impl_->GetRawORCReader(); }
+
 Result<std::shared_ptr<const KeyValueMetadata>> ORCFileReader::ReadMetadata() {
 return impl_->ReadMetadata();
 }
@@ -653,6 +680,11 @@ Result<std::shared_ptr<RecordBatchReader>> ORCFileReader::NextStripeReader(
 return reader;
@@ -605,6 +631,11 @@ Result<std::shared_ptr<RecordBatchReader>> ORCFileReader::NextStripeReader(
 return impl_->NextStripeReader(batch_size, include_indices);
 }

+Result<std::shared_ptr<RecordBatchReader>> ORCFileReader::NextStripeReader(
+ int64_t batch_size, const std::vector<std::string>& include_names) {
+ return impl_->NextStripeReader(batch_size, include_names);
+}
+
 int64_t ORCFileReader::NumberOfStripes() { return impl_->NumberOfStripes(); }

 int64_t ORCFileReader::NumberOfRows() { return impl_->NumberOfRows(); }
diff --git a/cpp/src/arrow/adapters/orc/adapter.h b/cpp/src/arrow/adapters/orc/adapter.h
index 223efa515..04e6b0612 100644
index 013be7860..7fd06bcb8 100644
--- a/cpp/src/arrow/adapters/orc/adapter.h
+++ b/cpp/src/arrow/adapters/orc/adapter.h
@@ -30,6 +30,10 @@
Expand All@@ -87,23 +86,23 @@ index 223efa515..04e6b0612 100644
+
+// alias to not interfere with nested orc namespace
+namespace liborc = orc;

 namespace arrow {
 namespace adapters {
@@ -51,6 +55,9 @@ class ARROW_EXPORT ORCFileReader {
 static Status Open(const std::shared_ptr<io::RandomAccessFile>& file, MemoryPool* pool,
 std::unique_ptr<ORCFileReader>* reader);
@@ -53,6 +57,9 @@ class ARROW_EXPORT ORCFileReader {
public:
~ORCFileReader();

+ /// \brief Get ORC reader from inside.
+ liborc::Reader* GetRawORCReader();
+
 /// \brief Creates a new ORC reader
 ///
 /// \param[in] file the data source
@@ -240,6 +247,19 @@ class ARROW_EXPORT ORCFileReader {
 Result<std::shared_ptr<RecordBatchReader>> NextStripeReader(
 int64_t batch_size, const std::vector<int>& include_indices);
@@ -174,6 +181,19 @@ class ARROW_EXPORT ORCFileReader {
 Result<std::shared_ptr<RecordBatchReader>> GetRecordBatchReader(
 int64_t batch_size, const std::vector<std::string>& include_names);

+ /// \brief Get a stripe level record batch iterator with specified row count
+ /// in each record batch. NextStripeReader serves as a fine grain
+ /// alternative to ReadStripe which may cause OOM issue by loading
Expand All@@ -119,4 +118,3 @@ index 223efa515..04e6b0612 100644
+
 /// \brief The number of stripes in the file
 int64_t NumberOfStripes();


41 changes: 28 additions & 13 deletions
thirdparty/vars.sh

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -238,11 +238,24 @@ FLATBUFFERS_NAME=flatbuffers-2.0.0.tar.gz
FLATBUFFERS_SOURCE=flatbuffers-2.0.0
FLATBUFFERS_MD5SUM="a27992324c3cbf86dd888268a23d17bd"

# c-ares
CARES_DOWNLOAD="https://github.com/c-ares/c-ares/releases/download/cares-1_19_1/c-ares-1.19.1.tar.gz"
CARES_NAME="c-ares-1.19.1.tar.gz"
CARES_SOURCE=c-ares-1.19.1
CARES_MD5SUM="dafc5825a92dc907e144570e4e75a908"

# grpc
# grpc v1.55 and above require protobuf >= 22
GRPC_DOWNLOAD="https://github.com/grpc/grpc/archive/refs/tags/v1.54.3.tar.gz"
GRPC_NAME="grpc-v1.54.3.tar.gz"
GRPC_SOURCE=grpc-1.54.3
GRPC_MD5SUM="af00a2edeae0f02bb25917cc3473b7de"

# arrow
ARROW_DOWNLOAD="https://archive.apache.org/dist/arrow/arrow-7.0.0/apache-arrow-7.0.0.tar.gz"
ARROW_NAME="apache-arrow-7.0.0.tar.gz"
ARROW_SOURCE="apache-arrow-7.0.0"
ARROW_MD5SUM="316ade159901646849b3b4760fa52816"
ARROW_DOWNLOAD="https://github.com/apache/arrow/archive/refs/tags/apache-arrow-13.0.0.tar.gz"
ARROW_NAME="apache-arrow-13.0.0.tar.gz"
ARROW_SOURCE="arrow-apache-arrow-13.0.0"
ARROW_MD5SUM="8ec1ec6a119514bcaea1cf7aabc9df1f"

# Abseil
ABSEIL_DOWNLOAD="https://github.com/abseil/abseil-cpp/archive/refs/tags/20220623.1.tar.gz"
Expand DownExpand Up@@ -287,10 +300,10 @@ ORC_SOURCE=orc-1.7.2
ORC_MD5SUM="6cab37935eacdec7d078d327746a8578"

# jemalloc for arrow
JEMALLOC_ARROW_DOWNLOAD="https://github.com/jemalloc/jemalloc/releases/download/5.2.1/jemalloc-5.2.1.tar.bz2"
JEMALLOC_ARROW_NAME="jemalloc-5.2.1.tar.bz2"
JEMALLOC_ARROW_SOURCE="jemalloc-5.2.1"
JEMALLOC_ARROW_MD5SUM="3d41fbf006e6ebffd489bdb304d009ae"
JEMALLOC_ARROW_DOWNLOAD="https://github.com/jemalloc/jemalloc/releases/download/5.3.0/jemalloc-5.3.0.tar.bz2"
JEMALLOC_ARROW_NAME="jemalloc-5.3.0.tar.bz2"
JEMALLOC_ARROW_SOURCE="jemalloc-5.3.0"
JEMALLOC_ARROW_MD5SUM="09a8328574dab22a7df848eae6dbbf53"

# jemalloc for doris
JEMALLOC_DORIS_DOWNLOAD="https://github.com/jemalloc/jemalloc/releases/download/5.3.0/jemalloc-5.3.0.tar.bz2"
Expand DownExpand Up@@ -398,11 +411,11 @@ BENCHMARK_SOURCE=benchmark-1.8.0
BENCHMARK_MD5SUM="8ddf8571d3f6198d37852bcbd964f817"

# xsimd
# for arrow-7.0.0, if arrow upgrade, this version may also need to be changed
XSIMD_DOWNLOAD="https://github.com/xtensor-stack/xsimd/archive/aeec9c872c8b475dedd7781336710f2dd2666cb2.tar.gz"
XSIMD_NAME=xsimd-aeec9c872c8b475dedd7781336710f2dd2666cb2.tar.gz
XSIMD_SOURCE=xsimd-aeec9c872c8b475dedd7781336710f2dd2666cb2
XSIMD_MD5SUM="d024855f71c0a2837a6918c0f8f66245"
# for arrow-13.0.0, if arrow upgrade, this version may also need to be changed
XSIMD_DOWNLOAD="https://github.com/xtensor-stack/xsimd/archive/refs/tags/9.0.1.tar.gz"
XSIMD_NAME="xsimd-9.0.1.tar.gz"
XSIMD_SOURCE=xsimd-9.0.1
XSIMD_MD5SUM="59f38fe3364acd7ed137771258812d6c"

# simdjson
SIMDJSON_DOWNLOAD="https://github.com/simdjson/simdjson/archive/refs/tags/v3.0.1.tar.gz"
Expand DownExpand Up@@ -505,6 +518,8 @@ export TP_ARCHIVES=(
 'CYRUS_SASL'
 'LIBRDKAFKA'
 'FLATBUFFERS'
 'CARES'
 'GRPC'
 'ARROW'
 'BROTLI'
 'ZSTD'
Expand Down

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Auto-enable theater mode on YouTube\n(function() {\n function tryTheater() {\n var btn = document.querySelector('button[aria-label=\"Theater mode\"], ytd-player #player button[title=\"Theater mode\"]');\n if (btn && !btn.classList.contains('activated')) {\n btn.click();\n }\n }\n \n // Try immediately\n tryTheater();\n \n // Try after navigation (SPA)\n var lastUrl = location.href;\n setInterval(function() {\n if (location.href !== lastUrl) {\n lastUrl = location.href;\n setTimeout(tryTheater, 500);\n }\n }, 1000);\n \n // Also try on player load\n var observer = new MutationObserver(tryTheater);\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "YouTube Theater Mode Default");
}
} catch(__e) { console.warn('[Userscript:YouTube Theater Mode Default]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
[Fix](Outfile) upgrade apache-arrow version to 13.0.0#35142

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Closed

BePPPower
wants to merge
3
commits into

from

 Closed


[Fix](Outfile) upgrade apache-arrow version to 13.0.0
#35142
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


3 commits


Select commit
Hold shift + click to select a range
466bddc

upgrade


BePPPower May 21, 2024
1d2137c

fix


BePPPower May 22, 2024
ff07338

fix 2


BePPPower May 22, 2024

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


7 changes: 7 additions & 0 deletions
thirdparty/CHANGELOG.md

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -170,6 +170,13 @@ Now there will be 2 set of libhdfs, one is without kerberos, the other is with k

## v20211215

## v20240521
- Modified: arrow 7.0.0 -> 13.0.0
- Modified: jemalloc for arrow 5.2.1 -> 5.3.0
- Modified: xsimd 7.0.0 -> 13.0.0
- Added: c-ares -> 1.19.1
- Added: grpc -> 1.54.3

### Changes

- Added: cyrus-sasl
Expand Down

72 changes: 71 additions & 1 deletion
thirdparty/build-thirdparty.sh

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -958,6 +958,8 @@ build_arrow() {
 export ARROW_ZLIB_URL="${TP_SOURCE_DIR}/${ZLIB_NAME}"
 export ARROW_XSIMD_URL="${TP_SOURCE_DIR}/${XSIMD_NAME}"
 export ARROW_ORC_URL="${TP_SOURCE_DIR}/${ORC_NAME}"
 export ARROW_GRPC_URL="${TP_SOURCE_DIR}/${GRPC_NAME}"
 export ARROW_PROTOBUF_URL="${TP_SOURCE_DIR}/${PROTOBUF_NAME}"

 if [[ "${KERNEL}" != 'Darwin' ]]; then
 ldflags="-L${TP_LIB_DIR} -static-libstdc++ -static-libgcc"
Expand All@@ -973,22 +975,38 @@ build_arrow() {
 -DCMAKE_INSTALL_PREFIX="${TP_INSTALL_DIR}" \
 -DCMAKE_INSTALL_LIBDIR=lib64 \
 -DARROW_BOOST_USE_SHARED=OFF \
 -DARROW_WITH_GRPC=ON \
 -DgRPC_SOURCE=SYSTEM \
 -DgRPC_ROOT="${TP_INSTALL_DIR}" \
 -DARROW_WITH_PROTOBUF=ON \
 -DProtobuf_SOURCE=SYSTEM \
 -DProtobuf_LIB="${TP_INSTALL_DIR}/lib/libprotoc.a" -DProtobuf_INCLUDE_DIR="${TP_INSTALL_DIR}/include" \
 -DARROW_FLIGHT=ON \
 -DARROW_FLIGHT_SQL=ON \
 -DBoost_USE_STATIC_RUNTIME=ON \
 -DARROW_GFLAGS_USE_SHARED=OFF \
 -Dgflags_ROOT="${TP_INSTALL_DIR}" \
 -DGLOG_ROOT="${TP_INSTALL_DIR}" \
 -DRE2_ROOT="${TP_INSTALL_DIR}" \
 -DZLIB_SOURCE=SYSTEM \
 -DZLIB_LIBRARY="${TP_INSTALL_DIR}/lib/libz.a" -DZLIB_INCLUDE_DIR="${TP_INSTALL_DIR}/include" \
 -DRapidJSON_SOURCE=SYSTEM \
 -DRapidJSON_ROOT="${TP_INSTALL_DIR}" \
 -DORC_ROOT="${TP_INSTALL_DIR}" \
 -Dxsimd_SOURCE=BUNDLED \
 -DBrotli_SOURCE=BUNDLED \
 -DARROW_LZ4_USE_SHARED=OFF \
 -DLZ4_LIB="${TP_INSTALL_DIR}/lib/liblz4.a" -DLZ4_INCLUDE_DIR="${TP_INSTALL_DIR}/include/lz4" \
 -DLz4_SOURCE=SYSTEM \
 -DARROW_ZSTD_USE_SHARED=OFF \
 -DZSTD_LIB="${TP_INSTALL_DIR}/lib/libzstd.a" -DZSTD_INCLUDE_DIR="${TP_INSTALL_DIR}/include" \
 -Dzstd_SOURCE=SYSTEM \
 -DSnappy_LIB="${TP_INSTALL_DIR}/lib/libsnappy.a" -DSnappy_INCLUDE_DIR="${TP_INSTALL_DIR}/include" \
 -DSnappy_SOURCE=SYSTEM \
 -DBOOST_ROOT="${TP_INSTALL_DIR}" --no-warn-unused-cli \
 -Djemalloc_SOURCE=BUNDLED \
 -DARROW_THRIFT_USE_SHARED=OFF \
 -DThrift_SOURCE=SYSTEM \
 -DThrift_ROOT="${TP_INSTALL_DIR}" ..

 "${BUILD_SYSTEM}" -j "${PARALLEL}"
Expand DownExpand Up@@ -1662,6 +1680,56 @@ build_libdeflate() {
 "${BUILD_SYSTEM}" install
}

# c-ares
build_cares() {
 check_if_source_exist "${CARES_SOURCE}"
 cd "${TP_SOURCE_DIR}/${CARES_SOURCE}"

 mkdir -p build
 cd build
 cmake -DCMAKE_BUILD_TYPE=Release \
 -DCARES_STATIC=ON \
 -DCARES_SHARED=OFF \
 -DCARES_STATIC_PIC=ON \
 -DCMAKE_INSTALL_PREFIX="${TP_INSTALL_DIR}" ..
 make
 make install
}

# grpc
build_grpc() {
 check_if_source_exist "${GRPC_SOURCE}"
 cd "${TP_SOURCE_DIR}/${GRPC_SOURCE}"

 mkdir -p cmake/build
 cd cmake/build

 cmake -DgRPC_INSTALL=ON \
 -DgRPC_BUILD_TESTS=OFF \
 -DCMAKE_BUILD_TYPE=Release \
 -DCMAKE_INSTALL_PREFIX="${TP_INSTALL_DIR}" \
 -DgRPC_CARES_PROVIDER=package \
 -Dc-ares_DIR="${TP_INSTALL_DIR}" \
 -DgRPC_ABSL_PROVIDER=package \
 -Dabsl_DIR="${TP_INSTALL_DIR}" \
 -DgRPC_PROTOBUF_PROVIDER=package \
 -DProtobuf_DIR="${TP_INSTALL_DIR}" \
 -DgRPC_RE2_PROVIDER=package \
 -Dre2_DIR:STRING="${TP_INSTALL_DIR}" \
 -DgRPC_SSL_PROVIDER=package \
 -DOPENSSL_ROOT_DIR="${TP_INSTALL_DIR}" \
 -DgRPC_ZLIB_PROVIDER=package \
 -DZLIB_ROOT="${TP_INSTALL_DIR}" \
 -DCMAKE_POSITION_INDEPENDENT_CODE=ON \
 ../..

 make -j "${PARALLEL}"
 make install

 # for grpc > v1.55, cmake 2.22 does not support find_dependency, delete this line after cmake version upgrade.
 # sed -i 's/find_dependency/find_package/g' "${TP_INSTALL_DIR}"/lib64/cmake/grpc/gRPCConfig.cmake
}

if [[ "${#packages[@]}" -eq 0 ]]; then
 packages=(
 libunixodbc
Expand All@@ -1673,9 +1741,9 @@ if [[ "${#packages[@]}" -eq 0 ]]; then
 lzo2
 zstd
 boost # must before thrift
 protobuf
 gflags
 gtest
 protobuf # after gtest
 glog
 rapidjson
 snappy
Expand All@@ -1693,6 +1761,8 @@ if [[ "${#packages[@]}" -eq 0 ]]; then
 librdkafka
 flatbuffers
 orc
 cares
 grpc # after cares, protobuf
 arrow
 abseil
 s2
Expand Down

4 changes: 2 additions & 2 deletions
thirdparty/download-thirdparty.sh

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -323,10 +323,10 @@ fi
echo "Finished patching ${OPENTELEMETRY_SOURCE}"

# arrow patch is used to get the raw orc reader for filter prune.
if [[ "${ARROW_SOURCE}" == "apache-arrow-7.0.0" ]]; then
if [[ "${ARROW_SOURCE}" == "arrow-apache-arrow-13.0.0" ]]; then
 cd "${TP_SOURCE_DIR}/${ARROW_SOURCE}"
 if [[ ! -f "${PATCHED_MARK}" ]]; then
 patch -p1 <"${TP_PATCH_DIR}/apache-arrow-7.0.0.patch"
 patch -p1 <"${TP_PATCH_DIR}/apache-arrow-13.0.0.patch"
 touch "${PATCHED_MARK}"
 fi
 cd -
Expand Down

58 changes: 28 additions & 30 deletions
thirdparty/patches/apache-arrow-7.0.0.patch → thirdparty/patches/apache-arrow-13.0.0.patch

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -1,82 +1,81 @@
diff --git a/cpp/src/arrow/adapters/orc/adapter.cc b/cpp/src/arrow/adapters/orc/adapter.cc
index 03243e751..1eb9b2c81 100644
index 2466e7433..46b4402d4 100644
--- a/cpp/src/arrow/adapters/orc/adapter.cc
+++ b/cpp/src/arrow/adapters/orc/adapter.cc
@@ -47,9 +47,6 @@
 #include "arrow/util/visibility.h"
 #include "orc/Exceptions.hh"

-// alias to not interfere with nested orc namespace
-namespace liborc = orc;
-
 #define ORC_THROW_NOT_OK(s) \
 do { \
 Status _s = (s); \
@@ -198,6 +195,8 @@ class ORCFileReader::Impl {
@@ -202,6 +199,8 @@ class ORCFileReader::Impl {
 return Init();
 }

+ virtual liborc::Reader* GetRawORCReader() { return reader_.get(); }
+
 Status Init() {
 int64_t nstripes = reader_->getNumberOfStripes();
 stripes_.resize(nstripes);
@@ -504,6 +503,32 @@ class ORCFileReader::Impl {
@@ -479,6 +478,31 @@ class ORCFileReader::Impl {
 return Status::OK();
 }

+ Result<std::shared_ptr<RecordBatchReader>> NextStripeReader(
+ int64_t batch_size, const std::vector<std::string>& include_names) {
+ if (current_row_ >= NumberOfRows()) {
+ return nullptr;
+ }
+
+ liborc::RowReaderOptions opts;
+ liborc::RowReaderOptions opts = default_row_reader_options();
+ if (!include_names.empty()) {
+ RETURN_NOT_OK(SelectNames(&opts, include_names));
+ }
+ StripeInformation stripe_info({0, 0, 0, 0});
+ RETURN_NOT_OK(SelectStripeWithRowNumber(&opts, current_row_, &stripe_info));
+ std::shared_ptr<Schema> schema;
+ RETURN_NOT_OK(ReadSchema(opts, &schema));
+ ARROW_ASSIGN_OR_RAISE(auto schema, ReadSchema(opts));
+ std::unique_ptr<liborc::RowReader> row_reader;
+
+ ORC_BEGIN_CATCH_NOT_OK
+ row_reader = reader_->createRowReader(opts);
+ row_reader->seekToRow(current_row_);
+ current_row_ = stripe_info.first_row_of_stripe + stripe_info.num_rows;
+ current_row_ = stripe_info.first_row_id + stripe_info.num_rows;
+ ORC_END_CATCH_NOT_OK
+
+ return std::make_shared<OrcStripeReader>(std::move(row_reader), schema, batch_size,
+ pool_);
+ }
+
Status NextStripeReader(int64_t batch_size, std::shared_ptr<RecordBatchReader>* out) {
return NextStripeReader(batch_size, {}, out);
}
@@ -531,6 +556,8 @@ Result<std::unique_ptr<ORCFileReader>> ORCFileReader::Open(
Result<std::shared_ptr<RecordBatchReader>> NextStripeReader(
 int64_t batch_size, const std::vector<int>& include_indices) {
 if (current_row_ >= NumberOfRows()) {
@@ -544,6 +568,8 @@ Result<std::unique_ptr<ORCFileReader>> ORCFileReader::Open(
 return std::move(result);
 }

+liborc::Reader* ORCFileReader::GetRawORCReader() { return impl_->GetRawORCReader(); }
+
 Result<std::shared_ptr<const KeyValueMetadata>> ORCFileReader::ReadMetadata() {
 return impl_->ReadMetadata();
 }
@@ -653,6 +680,11 @@ Result<std::shared_ptr<RecordBatchReader>> ORCFileReader::NextStripeReader(
 return reader;
@@ -605,6 +631,11 @@ Result<std::shared_ptr<RecordBatchReader>> ORCFileReader::NextStripeReader(
 return impl_->NextStripeReader(batch_size, include_indices);
 }

+Result<std::shared_ptr<RecordBatchReader>> ORCFileReader::NextStripeReader(
+ int64_t batch_size, const std::vector<std::string>& include_names) {
+ return impl_->NextStripeReader(batch_size, include_names);
+}
+
 int64_t ORCFileReader::NumberOfStripes() { return impl_->NumberOfStripes(); }

 int64_t ORCFileReader::NumberOfRows() { return impl_->NumberOfRows(); }
diff --git a/cpp/src/arrow/adapters/orc/adapter.h b/cpp/src/arrow/adapters/orc/adapter.h
index 223efa515..04e6b0612 100644
index 013be7860..7fd06bcb8 100644
--- a/cpp/src/arrow/adapters/orc/adapter.h
+++ b/cpp/src/arrow/adapters/orc/adapter.h
@@ -30,6 +30,10 @@
Expand All@@ -87,23 +86,23 @@ index 223efa515..04e6b0612 100644
+
+// alias to not interfere with nested orc namespace
+namespace liborc = orc;

 namespace arrow {
 namespace adapters {
@@ -51,6 +55,9 @@ class ARROW_EXPORT ORCFileReader {
 static Status Open(const std::shared_ptr<io::RandomAccessFile>& file, MemoryPool* pool,
 std::unique_ptr<ORCFileReader>* reader);
@@ -53,6 +57,9 @@ class ARROW_EXPORT ORCFileReader {
public:
~ORCFileReader();

+ /// \brief Get ORC reader from inside.
+ liborc::Reader* GetRawORCReader();
+
 /// \brief Creates a new ORC reader
 ///
 /// \param[in] file the data source
@@ -240,6 +247,19 @@ class ARROW_EXPORT ORCFileReader {
 Result<std::shared_ptr<RecordBatchReader>> NextStripeReader(
 int64_t batch_size, const std::vector<int>& include_indices);
@@ -174,6 +181,19 @@ class ARROW_EXPORT ORCFileReader {
 Result<std::shared_ptr<RecordBatchReader>> GetRecordBatchReader(
 int64_t batch_size, const std::vector<std::string>& include_names);

+ /// \brief Get a stripe level record batch iterator with specified row count
+ /// in each record batch. NextStripeReader serves as a fine grain
+ /// alternative to ReadStripe which may cause OOM issue by loading
Expand All@@ -119,4 +118,3 @@ index 223efa515..04e6b0612 100644
+
 /// \brief The number of stripes in the file
 int64_t NumberOfStripes();


41 changes: 28 additions & 13 deletions
thirdparty/vars.sh

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -238,11 +238,24 @@ FLATBUFFERS_NAME=flatbuffers-2.0.0.tar.gz
FLATBUFFERS_SOURCE=flatbuffers-2.0.0
FLATBUFFERS_MD5SUM="a27992324c3cbf86dd888268a23d17bd"

# c-ares
CARES_DOWNLOAD="https://github.com/c-ares/c-ares/releases/download/cares-1_19_1/c-ares-1.19.1.tar.gz"
CARES_NAME="c-ares-1.19.1.tar.gz"
CARES_SOURCE=c-ares-1.19.1
CARES_MD5SUM="dafc5825a92dc907e144570e4e75a908"

# grpc
# grpc v1.55 and above require protobuf >= 22
GRPC_DOWNLOAD="https://github.com/grpc/grpc/archive/refs/tags/v1.54.3.tar.gz"
GRPC_NAME="grpc-v1.54.3.tar.gz"
GRPC_SOURCE=grpc-1.54.3
GRPC_MD5SUM="af00a2edeae0f02bb25917cc3473b7de"

# arrow
ARROW_DOWNLOAD="https://archive.apache.org/dist/arrow/arrow-7.0.0/apache-arrow-7.0.0.tar.gz"
ARROW_NAME="apache-arrow-7.0.0.tar.gz"
ARROW_SOURCE="apache-arrow-7.0.0"
ARROW_MD5SUM="316ade159901646849b3b4760fa52816"
ARROW_DOWNLOAD="https://github.com/apache/arrow/archive/refs/tags/apache-arrow-13.0.0.tar.gz"
ARROW_NAME="apache-arrow-13.0.0.tar.gz"
ARROW_SOURCE="arrow-apache-arrow-13.0.0"
ARROW_MD5SUM="8ec1ec6a119514bcaea1cf7aabc9df1f"

# Abseil
ABSEIL_DOWNLOAD="https://github.com/abseil/abseil-cpp/archive/refs/tags/20220623.1.tar.gz"
Expand DownExpand Up@@ -287,10 +300,10 @@ ORC_SOURCE=orc-1.7.2
ORC_MD5SUM="6cab37935eacdec7d078d327746a8578"

# jemalloc for arrow
JEMALLOC_ARROW_DOWNLOAD="https://github.com/jemalloc/jemalloc/releases/download/5.2.1/jemalloc-5.2.1.tar.bz2"
JEMALLOC_ARROW_NAME="jemalloc-5.2.1.tar.bz2"
JEMALLOC_ARROW_SOURCE="jemalloc-5.2.1"
JEMALLOC_ARROW_MD5SUM="3d41fbf006e6ebffd489bdb304d009ae"
JEMALLOC_ARROW_DOWNLOAD="https://github.com/jemalloc/jemalloc/releases/download/5.3.0/jemalloc-5.3.0.tar.bz2"
JEMALLOC_ARROW_NAME="jemalloc-5.3.0.tar.bz2"
JEMALLOC_ARROW_SOURCE="jemalloc-5.3.0"
JEMALLOC_ARROW_MD5SUM="09a8328574dab22a7df848eae6dbbf53"

# jemalloc for doris
JEMALLOC_DORIS_DOWNLOAD="https://github.com/jemalloc/jemalloc/releases/download/5.3.0/jemalloc-5.3.0.tar.bz2"
Expand DownExpand Up@@ -398,11 +411,11 @@ BENCHMARK_SOURCE=benchmark-1.8.0
BENCHMARK_MD5SUM="8ddf8571d3f6198d37852bcbd964f817"

# xsimd
# for arrow-7.0.0, if arrow upgrade, this version may also need to be changed
XSIMD_DOWNLOAD="https://github.com/xtensor-stack/xsimd/archive/aeec9c872c8b475dedd7781336710f2dd2666cb2.tar.gz"
XSIMD_NAME=xsimd-aeec9c872c8b475dedd7781336710f2dd2666cb2.tar.gz
XSIMD_SOURCE=xsimd-aeec9c872c8b475dedd7781336710f2dd2666cb2
XSIMD_MD5SUM="d024855f71c0a2837a6918c0f8f66245"
# for arrow-13.0.0, if arrow upgrade, this version may also need to be changed
XSIMD_DOWNLOAD="https://github.com/xtensor-stack/xsimd/archive/refs/tags/9.0.1.tar.gz"
XSIMD_NAME="xsimd-9.0.1.tar.gz"
XSIMD_SOURCE=xsimd-9.0.1
XSIMD_MD5SUM="59f38fe3364acd7ed137771258812d6c"

# simdjson
SIMDJSON_DOWNLOAD="https://github.com/simdjson/simdjson/archive/refs/tags/v3.0.1.tar.gz"
Expand DownExpand Up@@ -505,6 +518,8 @@ export TP_ARCHIVES=(
 'CYRUS_SASL'
 'LIBRDKAFKA'
 'FLATBUFFERS'
 'CARES'
 'GRPC'
 'ARROW'
 'BROTLI'
 'ZSTD'
Expand Down

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Remove or un-stick sticky/fixed headers that block content\n(function() {\n function unstick() {\n document.querySelectorAll('header, nav, [role=\"banner\"], .header, .navbar, .sticky, .fixed-top, [style*=\"position: fixed\"], [style*=\"position:sticky\"]').forEach(function(el) {\n if (el.style.position === 'fixed' || el.style.position === 'sticky' || \n getComputedStyle(el).position === 'fixed' || getComputedStyle(el).position === 'sticky') {\n el.style.position = 'static';\n el.style.top = 'auto';\n el.style.zIndex = 'auto';\n }\n });\n }\n \n unstick();\n \n var observer = new MutationObserver(unstick);\n observer.observe(document.body, { childList: true, subtree: true, attributes: true, attributeFilter: ['style', 'class'] });\n})();", "Kill Sticky Headers");
}
} catch(__e) { console.warn('[Userscript:Kill Sticky Headers]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
[Fix](Outfile) upgrade apache-arrow version to 13.0.0#35142

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Closed

BePPPower
wants to merge
3
commits into

from

 Closed


[Fix](Outfile) upgrade apache-arrow version to 13.0.0
#35142
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


3 commits


Select commit
Hold shift + click to select a range
466bddc

upgrade


BePPPower May 21, 2024
1d2137c

fix


BePPPower May 22, 2024
ff07338

fix 2


BePPPower May 22, 2024

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


7 changes: 7 additions & 0 deletions
thirdparty/CHANGELOG.md

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -170,6 +170,13 @@ Now there will be 2 set of libhdfs, one is without kerberos, the other is with k

## v20211215

## v20240521
- Modified: arrow 7.0.0 -> 13.0.0
- Modified: jemalloc for arrow 5.2.1 -> 5.3.0
- Modified: xsimd 7.0.0 -> 13.0.0
- Added: c-ares -> 1.19.1
- Added: grpc -> 1.54.3

### Changes

- Added: cyrus-sasl
Expand Down

72 changes: 71 additions & 1 deletion
thirdparty/build-thirdparty.sh

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -958,6 +958,8 @@ build_arrow() {
 export ARROW_ZLIB_URL="${TP_SOURCE_DIR}/${ZLIB_NAME}"
 export ARROW_XSIMD_URL="${TP_SOURCE_DIR}/${XSIMD_NAME}"
 export ARROW_ORC_URL="${TP_SOURCE_DIR}/${ORC_NAME}"
 export ARROW_GRPC_URL="${TP_SOURCE_DIR}/${GRPC_NAME}"
 export ARROW_PROTOBUF_URL="${TP_SOURCE_DIR}/${PROTOBUF_NAME}"

 if [[ "${KERNEL}" != 'Darwin' ]]; then
 ldflags="-L${TP_LIB_DIR} -static-libstdc++ -static-libgcc"
Expand All@@ -973,22 +975,38 @@ build_arrow() {
 -DCMAKE_INSTALL_PREFIX="${TP_INSTALL_DIR}" \
 -DCMAKE_INSTALL_LIBDIR=lib64 \
 -DARROW_BOOST_USE_SHARED=OFF \
 -DARROW_WITH_GRPC=ON \
 -DgRPC_SOURCE=SYSTEM \
 -DgRPC_ROOT="${TP_INSTALL_DIR}" \
 -DARROW_WITH_PROTOBUF=ON \
 -DProtobuf_SOURCE=SYSTEM \
 -DProtobuf_LIB="${TP_INSTALL_DIR}/lib/libprotoc.a" -DProtobuf_INCLUDE_DIR="${TP_INSTALL_DIR}/include" \
 -DARROW_FLIGHT=ON \
 -DARROW_FLIGHT_SQL=ON \
 -DBoost_USE_STATIC_RUNTIME=ON \
 -DARROW_GFLAGS_USE_SHARED=OFF \
 -Dgflags_ROOT="${TP_INSTALL_DIR}" \
 -DGLOG_ROOT="${TP_INSTALL_DIR}" \
 -DRE2_ROOT="${TP_INSTALL_DIR}" \
 -DZLIB_SOURCE=SYSTEM \
 -DZLIB_LIBRARY="${TP_INSTALL_DIR}/lib/libz.a" -DZLIB_INCLUDE_DIR="${TP_INSTALL_DIR}/include" \
 -DRapidJSON_SOURCE=SYSTEM \
 -DRapidJSON_ROOT="${TP_INSTALL_DIR}" \
 -DORC_ROOT="${TP_INSTALL_DIR}" \
 -Dxsimd_SOURCE=BUNDLED \
 -DBrotli_SOURCE=BUNDLED \
 -DARROW_LZ4_USE_SHARED=OFF \
 -DLZ4_LIB="${TP_INSTALL_DIR}/lib/liblz4.a" -DLZ4_INCLUDE_DIR="${TP_INSTALL_DIR}/include/lz4" \
 -DLz4_SOURCE=SYSTEM \
 -DARROW_ZSTD_USE_SHARED=OFF \
 -DZSTD_LIB="${TP_INSTALL_DIR}/lib/libzstd.a" -DZSTD_INCLUDE_DIR="${TP_INSTALL_DIR}/include" \
 -Dzstd_SOURCE=SYSTEM \
 -DSnappy_LIB="${TP_INSTALL_DIR}/lib/libsnappy.a" -DSnappy_INCLUDE_DIR="${TP_INSTALL_DIR}/include" \
 -DSnappy_SOURCE=SYSTEM \
 -DBOOST_ROOT="${TP_INSTALL_DIR}" --no-warn-unused-cli \
 -Djemalloc_SOURCE=BUNDLED \
 -DARROW_THRIFT_USE_SHARED=OFF \
 -DThrift_SOURCE=SYSTEM \
 -DThrift_ROOT="${TP_INSTALL_DIR}" ..

 "${BUILD_SYSTEM}" -j "${PARALLEL}"
Expand DownExpand Up@@ -1662,6 +1680,56 @@ build_libdeflate() {
 "${BUILD_SYSTEM}" install
}

# c-ares
build_cares() {
 check_if_source_exist "${CARES_SOURCE}"
 cd "${TP_SOURCE_DIR}/${CARES_SOURCE}"

 mkdir -p build
 cd build
 cmake -DCMAKE_BUILD_TYPE=Release \
 -DCARES_STATIC=ON \
 -DCARES_SHARED=OFF \
 -DCARES_STATIC_PIC=ON \
 -DCMAKE_INSTALL_PREFIX="${TP_INSTALL_DIR}" ..
 make
 make install
}

# grpc
build_grpc() {
 check_if_source_exist "${GRPC_SOURCE}"
 cd "${TP_SOURCE_DIR}/${GRPC_SOURCE}"

 mkdir -p cmake/build
 cd cmake/build

 cmake -DgRPC_INSTALL=ON \
 -DgRPC_BUILD_TESTS=OFF \
 -DCMAKE_BUILD_TYPE=Release \
 -DCMAKE_INSTALL_PREFIX="${TP_INSTALL_DIR}" \
 -DgRPC_CARES_PROVIDER=package \
 -Dc-ares_DIR="${TP_INSTALL_DIR}" \
 -DgRPC_ABSL_PROVIDER=package \
 -Dabsl_DIR="${TP_INSTALL_DIR}" \
 -DgRPC_PROTOBUF_PROVIDER=package \
 -DProtobuf_DIR="${TP_INSTALL_DIR}" \
 -DgRPC_RE2_PROVIDER=package \
 -Dre2_DIR:STRING="${TP_INSTALL_DIR}" \
 -DgRPC_SSL_PROVIDER=package \
 -DOPENSSL_ROOT_DIR="${TP_INSTALL_DIR}" \
 -DgRPC_ZLIB_PROVIDER=package \
 -DZLIB_ROOT="${TP_INSTALL_DIR}" \
 -DCMAKE_POSITION_INDEPENDENT_CODE=ON \
 ../..

 make -j "${PARALLEL}"
 make install

 # for grpc > v1.55, cmake 2.22 does not support find_dependency, delete this line after cmake version upgrade.
 # sed -i 's/find_dependency/find_package/g' "${TP_INSTALL_DIR}"/lib64/cmake/grpc/gRPCConfig.cmake
}

if [[ "${#packages[@]}" -eq 0 ]]; then
 packages=(
 libunixodbc
Expand All@@ -1673,9 +1741,9 @@ if [[ "${#packages[@]}" -eq 0 ]]; then
 lzo2
 zstd
 boost # must before thrift
 protobuf
 gflags
 gtest
 protobuf # after gtest
 glog
 rapidjson
 snappy
Expand All@@ -1693,6 +1761,8 @@ if [[ "${#packages[@]}" -eq 0 ]]; then
 librdkafka
 flatbuffers
 orc
 cares
 grpc # after cares, protobuf
 arrow
 abseil
 s2
Expand Down

4 changes: 2 additions & 2 deletions
thirdparty/download-thirdparty.sh

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -323,10 +323,10 @@ fi
echo "Finished patching ${OPENTELEMETRY_SOURCE}"

# arrow patch is used to get the raw orc reader for filter prune.
if [[ "${ARROW_SOURCE}" == "apache-arrow-7.0.0" ]]; then
if [[ "${ARROW_SOURCE}" == "arrow-apache-arrow-13.0.0" ]]; then
 cd "${TP_SOURCE_DIR}/${ARROW_SOURCE}"
 if [[ ! -f "${PATCHED_MARK}" ]]; then
 patch -p1 <"${TP_PATCH_DIR}/apache-arrow-7.0.0.patch"
 patch -p1 <"${TP_PATCH_DIR}/apache-arrow-13.0.0.patch"
 touch "${PATCHED_MARK}"
 fi
 cd -
Expand Down

58 changes: 28 additions & 30 deletions
thirdparty/patches/apache-arrow-7.0.0.patch → thirdparty/patches/apache-arrow-13.0.0.patch

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -1,82 +1,81 @@
diff --git a/cpp/src/arrow/adapters/orc/adapter.cc b/cpp/src/arrow/adapters/orc/adapter.cc
index 03243e751..1eb9b2c81 100644
index 2466e7433..46b4402d4 100644
--- a/cpp/src/arrow/adapters/orc/adapter.cc
+++ b/cpp/src/arrow/adapters/orc/adapter.cc
@@ -47,9 +47,6 @@
 #include "arrow/util/visibility.h"
 #include "orc/Exceptions.hh"

-// alias to not interfere with nested orc namespace
-namespace liborc = orc;
-
 #define ORC_THROW_NOT_OK(s) \
 do { \
 Status _s = (s); \
@@ -198,6 +195,8 @@ class ORCFileReader::Impl {
@@ -202,6 +199,8 @@ class ORCFileReader::Impl {
 return Init();
 }

+ virtual liborc::Reader* GetRawORCReader() { return reader_.get(); }
+
 Status Init() {
 int64_t nstripes = reader_->getNumberOfStripes();
 stripes_.resize(nstripes);
@@ -504,6 +503,32 @@ class ORCFileReader::Impl {
@@ -479,6 +478,31 @@ class ORCFileReader::Impl {
 return Status::OK();
 }

+ Result<std::shared_ptr<RecordBatchReader>> NextStripeReader(
+ int64_t batch_size, const std::vector<std::string>& include_names) {
+ if (current_row_ >= NumberOfRows()) {
+ return nullptr;
+ }
+
+ liborc::RowReaderOptions opts;
+ liborc::RowReaderOptions opts = default_row_reader_options();
+ if (!include_names.empty()) {
+ RETURN_NOT_OK(SelectNames(&opts, include_names));
+ }
+ StripeInformation stripe_info({0, 0, 0, 0});
+ RETURN_NOT_OK(SelectStripeWithRowNumber(&opts, current_row_, &stripe_info));
+ std::shared_ptr<Schema> schema;
+ RETURN_NOT_OK(ReadSchema(opts, &schema));
+ ARROW_ASSIGN_OR_RAISE(auto schema, ReadSchema(opts));
+ std::unique_ptr<liborc::RowReader> row_reader;
+
+ ORC_BEGIN_CATCH_NOT_OK
+ row_reader = reader_->createRowReader(opts);
+ row_reader->seekToRow(current_row_);
+ current_row_ = stripe_info.first_row_of_stripe + stripe_info.num_rows;
+ current_row_ = stripe_info.first_row_id + stripe_info.num_rows;
+ ORC_END_CATCH_NOT_OK
+
+ return std::make_shared<OrcStripeReader>(std::move(row_reader), schema, batch_size,
+ pool_);
+ }
+
Status NextStripeReader(int64_t batch_size, std::shared_ptr<RecordBatchReader>* out) {
return NextStripeReader(batch_size, {}, out);
}
@@ -531,6 +556,8 @@ Result<std::unique_ptr<ORCFileReader>> ORCFileReader::Open(
Result<std::shared_ptr<RecordBatchReader>> NextStripeReader(
 int64_t batch_size, const std::vector<int>& include_indices) {
 if (current_row_ >= NumberOfRows()) {
@@ -544,6 +568,8 @@ Result<std::unique_ptr<ORCFileReader>> ORCFileReader::Open(
 return std::move(result);
 }

+liborc::Reader* ORCFileReader::GetRawORCReader() { return impl_->GetRawORCReader(); }
+
 Result<std::shared_ptr<const KeyValueMetadata>> ORCFileReader::ReadMetadata() {
 return impl_->ReadMetadata();
 }
@@ -653,6 +680,11 @@ Result<std::shared_ptr<RecordBatchReader>> ORCFileReader::NextStripeReader(
 return reader;
@@ -605,6 +631,11 @@ Result<std::shared_ptr<RecordBatchReader>> ORCFileReader::NextStripeReader(
 return impl_->NextStripeReader(batch_size, include_indices);
 }

+Result<std::shared_ptr<RecordBatchReader>> ORCFileReader::NextStripeReader(
+ int64_t batch_size, const std::vector<std::string>& include_names) {
+ return impl_->NextStripeReader(batch_size, include_names);
+}
+
 int64_t ORCFileReader::NumberOfStripes() { return impl_->NumberOfStripes(); }

 int64_t ORCFileReader::NumberOfRows() { return impl_->NumberOfRows(); }
diff --git a/cpp/src/arrow/adapters/orc/adapter.h b/cpp/src/arrow/adapters/orc/adapter.h
index 223efa515..04e6b0612 100644
index 013be7860..7fd06bcb8 100644
--- a/cpp/src/arrow/adapters/orc/adapter.h
+++ b/cpp/src/arrow/adapters/orc/adapter.h
@@ -30,6 +30,10 @@
Expand All@@ -87,23 +86,23 @@ index 223efa515..04e6b0612 100644
+
+// alias to not interfere with nested orc namespace
+namespace liborc = orc;

 namespace arrow {
 namespace adapters {
@@ -51,6 +55,9 @@ class ARROW_EXPORT ORCFileReader {
 static Status Open(const std::shared_ptr<io::RandomAccessFile>& file, MemoryPool* pool,
 std::unique_ptr<ORCFileReader>* reader);
@@ -53,6 +57,9 @@ class ARROW_EXPORT ORCFileReader {
public:
~ORCFileReader();

+ /// \brief Get ORC reader from inside.
+ liborc::Reader* GetRawORCReader();
+
 /// \brief Creates a new ORC reader
 ///
 /// \param[in] file the data source
@@ -240,6 +247,19 @@ class ARROW_EXPORT ORCFileReader {
 Result<std::shared_ptr<RecordBatchReader>> NextStripeReader(
 int64_t batch_size, const std::vector<int>& include_indices);
@@ -174,6 +181,19 @@ class ARROW_EXPORT ORCFileReader {
 Result<std::shared_ptr<RecordBatchReader>> GetRecordBatchReader(
 int64_t batch_size, const std::vector<std::string>& include_names);

+ /// \brief Get a stripe level record batch iterator with specified row count
+ /// in each record batch. NextStripeReader serves as a fine grain
+ /// alternative to ReadStripe which may cause OOM issue by loading
Expand All@@ -119,4 +118,3 @@ index 223efa515..04e6b0612 100644
+
 /// \brief The number of stripes in the file
 int64_t NumberOfStripes();


41 changes: 28 additions & 13 deletions
thirdparty/vars.sh

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -238,11 +238,24 @@ FLATBUFFERS_NAME=flatbuffers-2.0.0.tar.gz
FLATBUFFERS_SOURCE=flatbuffers-2.0.0
FLATBUFFERS_MD5SUM="a27992324c3cbf86dd888268a23d17bd"

# c-ares
CARES_DOWNLOAD="https://github.com/c-ares/c-ares/releases/download/cares-1_19_1/c-ares-1.19.1.tar.gz"
CARES_NAME="c-ares-1.19.1.tar.gz"
CARES_SOURCE=c-ares-1.19.1
CARES_MD5SUM="dafc5825a92dc907e144570e4e75a908"

# grpc
# grpc v1.55 and above require protobuf >= 22
GRPC_DOWNLOAD="https://github.com/grpc/grpc/archive/refs/tags/v1.54.3.tar.gz"
GRPC_NAME="grpc-v1.54.3.tar.gz"
GRPC_SOURCE=grpc-1.54.3
GRPC_MD5SUM="af00a2edeae0f02bb25917cc3473b7de"

# arrow
ARROW_DOWNLOAD="https://archive.apache.org/dist/arrow/arrow-7.0.0/apache-arrow-7.0.0.tar.gz"
ARROW_NAME="apache-arrow-7.0.0.tar.gz"
ARROW_SOURCE="apache-arrow-7.0.0"
ARROW_MD5SUM="316ade159901646849b3b4760fa52816"
ARROW_DOWNLOAD="https://github.com/apache/arrow/archive/refs/tags/apache-arrow-13.0.0.tar.gz"
ARROW_NAME="apache-arrow-13.0.0.tar.gz"
ARROW_SOURCE="arrow-apache-arrow-13.0.0"
ARROW_MD5SUM="8ec1ec6a119514bcaea1cf7aabc9df1f"

# Abseil
ABSEIL_DOWNLOAD="https://github.com/abseil/abseil-cpp/archive/refs/tags/20220623.1.tar.gz"
Expand DownExpand Up@@ -287,10 +300,10 @@ ORC_SOURCE=orc-1.7.2
ORC_MD5SUM="6cab37935eacdec7d078d327746a8578"

# jemalloc for arrow
JEMALLOC_ARROW_DOWNLOAD="https://github.com/jemalloc/jemalloc/releases/download/5.2.1/jemalloc-5.2.1.tar.bz2"
JEMALLOC_ARROW_NAME="jemalloc-5.2.1.tar.bz2"
JEMALLOC_ARROW_SOURCE="jemalloc-5.2.1"
JEMALLOC_ARROW_MD5SUM="3d41fbf006e6ebffd489bdb304d009ae"
JEMALLOC_ARROW_DOWNLOAD="https://github.com/jemalloc/jemalloc/releases/download/5.3.0/jemalloc-5.3.0.tar.bz2"
JEMALLOC_ARROW_NAME="jemalloc-5.3.0.tar.bz2"
JEMALLOC_ARROW_SOURCE="jemalloc-5.3.0"
JEMALLOC_ARROW_MD5SUM="09a8328574dab22a7df848eae6dbbf53"

# jemalloc for doris
JEMALLOC_DORIS_DOWNLOAD="https://github.com/jemalloc/jemalloc/releases/download/5.3.0/jemalloc-5.3.0.tar.bz2"
Expand DownExpand Up@@ -398,11 +411,11 @@ BENCHMARK_SOURCE=benchmark-1.8.0
BENCHMARK_MD5SUM="8ddf8571d3f6198d37852bcbd964f817"

# xsimd
# for arrow-7.0.0, if arrow upgrade, this version may also need to be changed
XSIMD_DOWNLOAD="https://github.com/xtensor-stack/xsimd/archive/aeec9c872c8b475dedd7781336710f2dd2666cb2.tar.gz"
XSIMD_NAME=xsimd-aeec9c872c8b475dedd7781336710f2dd2666cb2.tar.gz
XSIMD_SOURCE=xsimd-aeec9c872c8b475dedd7781336710f2dd2666cb2
XSIMD_MD5SUM="d024855f71c0a2837a6918c0f8f66245"
# for arrow-13.0.0, if arrow upgrade, this version may also need to be changed
XSIMD_DOWNLOAD="https://github.com/xtensor-stack/xsimd/archive/refs/tags/9.0.1.tar.gz"
XSIMD_NAME="xsimd-9.0.1.tar.gz"
XSIMD_SOURCE=xsimd-9.0.1
XSIMD_MD5SUM="59f38fe3364acd7ed137771258812d6c"

# simdjson
SIMDJSON_DOWNLOAD="https://github.com/simdjson/simdjson/archive/refs/tags/v3.0.1.tar.gz"
Expand DownExpand Up@@ -505,6 +518,8 @@ export TP_ARCHIVES=(
 'CYRUS_SASL'
 'LIBRDKAFKA'
 'FLATBUFFERS'
 'CARES'
 'GRPC'
 'ARROW'
 'BROTLI'
 'ZSTD'
Expand Down

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Universal Dark Mode - works on any site\n(function() {\n var enabled = true;\n \n function applyDarkMode() {\n if (!enabled) return;\n \n // Create style element if it doesn't exist\n var style = document.getElementById('universal-dark-mode-style');\n if (!style) {\n style = document.createElement('style');\n style.id = 'universal-dark-mode-style';\n document.head.appendChild(style);\n }\n \n // Dark mode CSS - inverts colors but preserves images/video\n style.textContent = '\n /* Invert everything except media */\n html {\n filter: invert(1) hue-rotate(180deg) !important;\n background: #1a1a2e !important;\n }\n \n /* Restore images, videos, iframes, canvas */\n img, video, iframe, canvas, svg, picture, [style*=\"background-image\"] {\n filter: invert(1) hue-rotate(180deg) !important;\n }\n \n /* Preserve specific elements that should not be inverted */\n .no-dark-mode, .no-dark-mode *,\n [data-theme=\"light\"], [data-theme=\"light\"],\n .ace_editor, .ace_editor *,\n .CodeMirror, .CodeMirror *,\n .monaco-editor, .monaco-editor *,\n .markdown-body pre, .markdown-body pre *,\n .highlight, .highlight *,\n pre code, pre code * {\n filter: none !important;\n }\n \n /* Fix common UI elements */\n .modal, .popup, .dropdown-menu, .tooltip, .popover {\n filter: invert(1) hue-rotate(180deg) !important;\n background: #2d2d44 !important;\n border-color: #444 !important;\n }\n \n /* Scrollbars */\n ::-webkit-scrollbar { background: #1a1a2e !important; }\n ::-webkit-scrollbar-thumb { background: #444 !important; }\n ::-webkit-scrollbar-thumb:hover { background: #555 !important; }\n \n /* Selection */\n ::selection { background: #4ecdc4 !important; color: #1a1a2e !important; }\n ::-moz-selection { background: #4ecdc4 !important; color: #1a1a2e !important; }\n ';\n }\n \n function removeDarkMode() {\n var style = document.getElementById('universal-dark-mode-style');\n if (style) style.remove();\n }\n \n // Toggle with Alt+Shift+D\n document.addEventListener('keydown', function(e) {\n if (e.altKey && e.shiftKey && e.key === 'D') {\n e.preventDefault();\n enabled = !enabled;\n if (enabled) {\n applyDarkMode();\n console.log('[Universal Dark Mode] Enabled');\n } else {\n removeDarkMode();\n console.log('[Universal Dark Mode] Disabled');\n }\n }\n });\n \n // Apply on load\n applyDarkMode();\n \n // Re-apply on dynamic content\n var observer = new MutationObserver(function(mutations) {\n if (enabled && !document.getElementById('universal-dark-mode-style')) {\n applyDarkMode();\n }\n });\n observer.observe(document.head, { childList: true });\n \n console.log('[Universal Dark Mode] Loaded - Press Alt+Shift+D to toggle');\n})();", "Universal Dark Mode");
}
} catch(__e) { console.warn('[Userscript:Universal Dark Mode]', __e); }
})();
})();

Skip to content
[Fix](Outfile) upgrade apache-arrow version to 13.0.0#35142

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Closed

BePPPower
wants to merge
3
commits into

from

 Closed


[Fix](Outfile) upgrade apache-arrow version to 13.0.0
#35142
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


3 commits


Select commit
Hold shift + click to select a range
466bddc

upgrade


BePPPower May 21, 2024
1d2137c

fix


BePPPower May 22, 2024
ff07338

fix 2


BePPPower May 22, 2024

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


7 changes: 7 additions & 0 deletions
thirdparty/CHANGELOG.md

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -170,6 +170,13 @@ Now there will be 2 set of libhdfs, one is without kerberos, the other is with k

## v20211215

## v20240521
- Modified: arrow 7.0.0 -> 13.0.0
- Modified: jemalloc for arrow 5.2.1 -> 5.3.0
- Modified: xsimd 7.0.0 -> 13.0.0
- Added: c-ares -> 1.19.1
- Added: grpc -> 1.54.3

### Changes

- Added: cyrus-sasl
Expand Down

72 changes: 71 additions & 1 deletion
thirdparty/build-thirdparty.sh

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -958,6 +958,8 @@ build_arrow() {
 export ARROW_ZLIB_URL="${TP_SOURCE_DIR}/${ZLIB_NAME}"
 export ARROW_XSIMD_URL="${TP_SOURCE_DIR}/${XSIMD_NAME}"
 export ARROW_ORC_URL="${TP_SOURCE_DIR}/${ORC_NAME}"
 export ARROW_GRPC_URL="${TP_SOURCE_DIR}/${GRPC_NAME}"
 export ARROW_PROTOBUF_URL="${TP_SOURCE_DIR}/${PROTOBUF_NAME}"

 if [[ "${KERNEL}" != 'Darwin' ]]; then
 ldflags="-L${TP_LIB_DIR} -static-libstdc++ -static-libgcc"
Expand All@@ -973,22 +975,38 @@ build_arrow() {
 -DCMAKE_INSTALL_PREFIX="${TP_INSTALL_DIR}" \
 -DCMAKE_INSTALL_LIBDIR=lib64 \
 -DARROW_BOOST_USE_SHARED=OFF \
 -DARROW_WITH_GRPC=ON \
 -DgRPC_SOURCE=SYSTEM \
 -DgRPC_ROOT="${TP_INSTALL_DIR}" \
 -DARROW_WITH_PROTOBUF=ON \
 -DProtobuf_SOURCE=SYSTEM \
 -DProtobuf_LIB="${TP_INSTALL_DIR}/lib/libprotoc.a" -DProtobuf_INCLUDE_DIR="${TP_INSTALL_DIR}/include" \
 -DARROW_FLIGHT=ON \
 -DARROW_FLIGHT_SQL=ON \
 -DBoost_USE_STATIC_RUNTIME=ON \
 -DARROW_GFLAGS_USE_SHARED=OFF \
 -Dgflags_ROOT="${TP_INSTALL_DIR}" \
 -DGLOG_ROOT="${TP_INSTALL_DIR}" \
 -DRE2_ROOT="${TP_INSTALL_DIR}" \
 -DZLIB_SOURCE=SYSTEM \
 -DZLIB_LIBRARY="${TP_INSTALL_DIR}/lib/libz.a" -DZLIB_INCLUDE_DIR="${TP_INSTALL_DIR}/include" \
 -DRapidJSON_SOURCE=SYSTEM \
 -DRapidJSON_ROOT="${TP_INSTALL_DIR}" \
 -DORC_ROOT="${TP_INSTALL_DIR}" \
 -Dxsimd_SOURCE=BUNDLED \
 -DBrotli_SOURCE=BUNDLED \
 -DARROW_LZ4_USE_SHARED=OFF \
 -DLZ4_LIB="${TP_INSTALL_DIR}/lib/liblz4.a" -DLZ4_INCLUDE_DIR="${TP_INSTALL_DIR}/include/lz4" \
 -DLz4_SOURCE=SYSTEM \
 -DARROW_ZSTD_USE_SHARED=OFF \
 -DZSTD_LIB="${TP_INSTALL_DIR}/lib/libzstd.a" -DZSTD_INCLUDE_DIR="${TP_INSTALL_DIR}/include" \
 -Dzstd_SOURCE=SYSTEM \
 -DSnappy_LIB="${TP_INSTALL_DIR}/lib/libsnappy.a" -DSnappy_INCLUDE_DIR="${TP_INSTALL_DIR}/include" \
 -DSnappy_SOURCE=SYSTEM \
 -DBOOST_ROOT="${TP_INSTALL_DIR}" --no-warn-unused-cli \
 -Djemalloc_SOURCE=BUNDLED \
 -DARROW_THRIFT_USE_SHARED=OFF \
 -DThrift_SOURCE=SYSTEM \
 -DThrift_ROOT="${TP_INSTALL_DIR}" ..

 "${BUILD_SYSTEM}" -j "${PARALLEL}"
Expand DownExpand Up@@ -1662,6 +1680,56 @@ build_libdeflate() {
 "${BUILD_SYSTEM}" install
}

# c-ares
build_cares() {
 check_if_source_exist "${CARES_SOURCE}"
 cd "${TP_SOURCE_DIR}/${CARES_SOURCE}"

 mkdir -p build
 cd build
 cmake -DCMAKE_BUILD_TYPE=Release \
 -DCARES_STATIC=ON \
 -DCARES_SHARED=OFF \
 -DCARES_STATIC_PIC=ON \
 -DCMAKE_INSTALL_PREFIX="${TP_INSTALL_DIR}" ..
 make
 make install
}

# grpc
build_grpc() {
 check_if_source_exist "${GRPC_SOURCE}"
 cd "${TP_SOURCE_DIR}/${GRPC_SOURCE}"

 mkdir -p cmake/build
 cd cmake/build

 cmake -DgRPC_INSTALL=ON \
 -DgRPC_BUILD_TESTS=OFF \
 -DCMAKE_BUILD_TYPE=Release \
 -DCMAKE_INSTALL_PREFIX="${TP_INSTALL_DIR}" \
 -DgRPC_CARES_PROVIDER=package \
 -Dc-ares_DIR="${TP_INSTALL_DIR}" \
 -DgRPC_ABSL_PROVIDER=package \
 -Dabsl_DIR="${TP_INSTALL_DIR}" \
 -DgRPC_PROTOBUF_PROVIDER=package \
 -DProtobuf_DIR="${TP_INSTALL_DIR}" \
 -DgRPC_RE2_PROVIDER=package \
 -Dre2_DIR:STRING="${TP_INSTALL_DIR}" \
 -DgRPC_SSL_PROVIDER=package \
 -DOPENSSL_ROOT_DIR="${TP_INSTALL_DIR}" \
 -DgRPC_ZLIB_PROVIDER=package \
 -DZLIB_ROOT="${TP_INSTALL_DIR}" \
 -DCMAKE_POSITION_INDEPENDENT_CODE=ON \
 ../..

 make -j "${PARALLEL}"
 make install

 # for grpc > v1.55, cmake 2.22 does not support find_dependency, delete this line after cmake version upgrade.
 # sed -i 's/find_dependency/find_package/g' "${TP_INSTALL_DIR}"/lib64/cmake/grpc/gRPCConfig.cmake
}

if [[ "${#packages[@]}" -eq 0 ]]; then
 packages=(
 libunixodbc
Expand All@@ -1673,9 +1741,9 @@ if [[ "${#packages[@]}" -eq 0 ]]; then
 lzo2
 zstd
 boost # must before thrift
 protobuf
 gflags
 gtest
 protobuf # after gtest
 glog
 rapidjson
 snappy
Expand All@@ -1693,6 +1761,8 @@ if [[ "${#packages[@]}" -eq 0 ]]; then
 librdkafka
 flatbuffers
 orc
 cares
 grpc # after cares, protobuf
 arrow
 abseil
 s2
Expand Down

4 changes: 2 additions & 2 deletions
thirdparty/download-thirdparty.sh

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -323,10 +323,10 @@ fi
echo "Finished patching ${OPENTELEMETRY_SOURCE}"

# arrow patch is used to get the raw orc reader for filter prune.
if [[ "${ARROW_SOURCE}" == "apache-arrow-7.0.0" ]]; then
if [[ "${ARROW_SOURCE}" == "arrow-apache-arrow-13.0.0" ]]; then
 cd "${TP_SOURCE_DIR}/${ARROW_SOURCE}"
 if [[ ! -f "${PATCHED_MARK}" ]]; then
 patch -p1 <"${TP_PATCH_DIR}/apache-arrow-7.0.0.patch"
 patch -p1 <"${TP_PATCH_DIR}/apache-arrow-13.0.0.patch"
 touch "${PATCHED_MARK}"
 fi
 cd -
Expand Down

58 changes: 28 additions & 30 deletions
thirdparty/patches/apache-arrow-7.0.0.patch → thirdparty/patches/apache-arrow-13.0.0.patch

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -1,82 +1,81 @@
diff --git a/cpp/src/arrow/adapters/orc/adapter.cc b/cpp/src/arrow/adapters/orc/adapter.cc
index 03243e751..1eb9b2c81 100644
index 2466e7433..46b4402d4 100644
--- a/cpp/src/arrow/adapters/orc/adapter.cc
+++ b/cpp/src/arrow/adapters/orc/adapter.cc
@@ -47,9 +47,6 @@
 #include "arrow/util/visibility.h"
 #include "orc/Exceptions.hh"

-// alias to not interfere with nested orc namespace
-namespace liborc = orc;
-
 #define ORC_THROW_NOT_OK(s) \
 do { \
 Status _s = (s); \
@@ -198,6 +195,8 @@ class ORCFileReader::Impl {
@@ -202,6 +199,8 @@ class ORCFileReader::Impl {
 return Init();
 }

+ virtual liborc::Reader* GetRawORCReader() { return reader_.get(); }
+
 Status Init() {
 int64_t nstripes = reader_->getNumberOfStripes();
 stripes_.resize(nstripes);
@@ -504,6 +503,32 @@ class ORCFileReader::Impl {
@@ -479,6 +478,31 @@ class ORCFileReader::Impl {
 return Status::OK();
 }

+ Result<std::shared_ptr<RecordBatchReader>> NextStripeReader(
+ int64_t batch_size, const std::vector<std::string>& include_names) {
+ if (current_row_ >= NumberOfRows()) {
+ return nullptr;
+ }
+
+ liborc::RowReaderOptions opts;
+ liborc::RowReaderOptions opts = default_row_reader_options();
+ if (!include_names.empty()) {
+ RETURN_NOT_OK(SelectNames(&opts, include_names));
+ }
+ StripeInformation stripe_info({0, 0, 0, 0});
+ RETURN_NOT_OK(SelectStripeWithRowNumber(&opts, current_row_, &stripe_info));
+ std::shared_ptr<Schema> schema;
+ RETURN_NOT_OK(ReadSchema(opts, &schema));
+ ARROW_ASSIGN_OR_RAISE(auto schema, ReadSchema(opts));
+ std::unique_ptr<liborc::RowReader> row_reader;
+
+ ORC_BEGIN_CATCH_NOT_OK
+ row_reader = reader_->createRowReader(opts);
+ row_reader->seekToRow(current_row_);
+ current_row_ = stripe_info.first_row_of_stripe + stripe_info.num_rows;
+ current_row_ = stripe_info.first_row_id + stripe_info.num_rows;
+ ORC_END_CATCH_NOT_OK
+
+ return std::make_shared<OrcStripeReader>(std::move(row_reader), schema, batch_size,
+ pool_);
+ }
+
Status NextStripeReader(int64_t batch_size, std::shared_ptr<RecordBatchReader>* out) {
return NextStripeReader(batch_size, {}, out);
}
@@ -531,6 +556,8 @@ Result<std::unique_ptr<ORCFileReader>> ORCFileReader::Open(
Result<std::shared_ptr<RecordBatchReader>> NextStripeReader(
 int64_t batch_size, const std::vector<int>& include_indices) {
 if (current_row_ >= NumberOfRows()) {
@@ -544,6 +568,8 @@ Result<std::unique_ptr<ORCFileReader>> ORCFileReader::Open(
 return std::move(result);
 }

+liborc::Reader* ORCFileReader::GetRawORCReader() { return impl_->GetRawORCReader(); }
+
 Result<std::shared_ptr<const KeyValueMetadata>> ORCFileReader::ReadMetadata() {
 return impl_->ReadMetadata();
 }
@@ -653,6 +680,11 @@ Result<std::shared_ptr<RecordBatchReader>> ORCFileReader::NextStripeReader(
 return reader;
@@ -605,6 +631,11 @@ Result<std::shared_ptr<RecordBatchReader>> ORCFileReader::NextStripeReader(
 return impl_->NextStripeReader(batch_size, include_indices);
 }

+Result<std::shared_ptr<RecordBatchReader>> ORCFileReader::NextStripeReader(
+ int64_t batch_size, const std::vector<std::string>& include_names) {
+ return impl_->NextStripeReader(batch_size, include_names);
+}
+
 int64_t ORCFileReader::NumberOfStripes() { return impl_->NumberOfStripes(); }

 int64_t ORCFileReader::NumberOfRows() { return impl_->NumberOfRows(); }
diff --git a/cpp/src/arrow/adapters/orc/adapter.h b/cpp/src/arrow/adapters/orc/adapter.h
index 223efa515..04e6b0612 100644
index 013be7860..7fd06bcb8 100644
--- a/cpp/src/arrow/adapters/orc/adapter.h
+++ b/cpp/src/arrow/adapters/orc/adapter.h
@@ -30,6 +30,10 @@
Expand All@@ -87,23 +86,23 @@ index 223efa515..04e6b0612 100644
+
+// alias to not interfere with nested orc namespace
+namespace liborc = orc;

 namespace arrow {
 namespace adapters {
@@ -51,6 +55,9 @@ class ARROW_EXPORT ORCFileReader {
 static Status Open(const std::shared_ptr<io::RandomAccessFile>& file, MemoryPool* pool,
 std::unique_ptr<ORCFileReader>* reader);
@@ -53,6 +57,9 @@ class ARROW_EXPORT ORCFileReader {
public:
~ORCFileReader();

+ /// \brief Get ORC reader from inside.
+ liborc::Reader* GetRawORCReader();
+
 /// \brief Creates a new ORC reader
 ///
 /// \param[in] file the data source
@@ -240,6 +247,19 @@ class ARROW_EXPORT ORCFileReader {
 Result<std::shared_ptr<RecordBatchReader>> NextStripeReader(
 int64_t batch_size, const std::vector<int>& include_indices);
@@ -174,6 +181,19 @@ class ARROW_EXPORT ORCFileReader {
 Result<std::shared_ptr<RecordBatchReader>> GetRecordBatchReader(
 int64_t batch_size, const std::vector<std::string>& include_names);

+ /// \brief Get a stripe level record batch iterator with specified row count
+ /// in each record batch. NextStripeReader serves as a fine grain
+ /// alternative to ReadStripe which may cause OOM issue by loading
Expand All@@ -119,4 +118,3 @@ index 223efa515..04e6b0612 100644
+
 /// \brief The number of stripes in the file
 int64_t NumberOfStripes();


41 changes: 28 additions & 13 deletions
thirdparty/vars.sh

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -238,11 +238,24 @@ FLATBUFFERS_NAME=flatbuffers-2.0.0.tar.gz
FLATBUFFERS_SOURCE=flatbuffers-2.0.0
FLATBUFFERS_MD5SUM="a27992324c3cbf86dd888268a23d17bd"

# c-ares
CARES_DOWNLOAD="https://github.com/c-ares/c-ares/releases/download/cares-1_19_1/c-ares-1.19.1.tar.gz"
CARES_NAME="c-ares-1.19.1.tar.gz"
CARES_SOURCE=c-ares-1.19.1
CARES_MD5SUM="dafc5825a92dc907e144570e4e75a908"

# grpc
# grpc v1.55 and above require protobuf >= 22
GRPC_DOWNLOAD="https://github.com/grpc/grpc/archive/refs/tags/v1.54.3.tar.gz"
GRPC_NAME="grpc-v1.54.3.tar.gz"
GRPC_SOURCE=grpc-1.54.3
GRPC_MD5SUM="af00a2edeae0f02bb25917cc3473b7de"

# arrow
ARROW_DOWNLOAD="https://archive.apache.org/dist/arrow/arrow-7.0.0/apache-arrow-7.0.0.tar.gz"
ARROW_NAME="apache-arrow-7.0.0.tar.gz"
ARROW_SOURCE="apache-arrow-7.0.0"
ARROW_MD5SUM="316ade159901646849b3b4760fa52816"
ARROW_DOWNLOAD="https://github.com/apache/arrow/archive/refs/tags/apache-arrow-13.0.0.tar.gz"
ARROW_NAME="apache-arrow-13.0.0.tar.gz"
ARROW_SOURCE="arrow-apache-arrow-13.0.0"
ARROW_MD5SUM="8ec1ec6a119514bcaea1cf7aabc9df1f"

# Abseil
ABSEIL_DOWNLOAD="https://github.com/abseil/abseil-cpp/archive/refs/tags/20220623.1.tar.gz"
Expand DownExpand Up@@ -287,10 +300,10 @@ ORC_SOURCE=orc-1.7.2
ORC_MD5SUM="6cab37935eacdec7d078d327746a8578"

# jemalloc for arrow
JEMALLOC_ARROW_DOWNLOAD="https://github.com/jemalloc/jemalloc/releases/download/5.2.1/jemalloc-5.2.1.tar.bz2"
JEMALLOC_ARROW_NAME="jemalloc-5.2.1.tar.bz2"
JEMALLOC_ARROW_SOURCE="jemalloc-5.2.1"
JEMALLOC_ARROW_MD5SUM="3d41fbf006e6ebffd489bdb304d009ae"
JEMALLOC_ARROW_DOWNLOAD="https://github.com/jemalloc/jemalloc/releases/download/5.3.0/jemalloc-5.3.0.tar.bz2"
JEMALLOC_ARROW_NAME="jemalloc-5.3.0.tar.bz2"
JEMALLOC_ARROW_SOURCE="jemalloc-5.3.0"
JEMALLOC_ARROW_MD5SUM="09a8328574dab22a7df848eae6dbbf53"

# jemalloc for doris
JEMALLOC_DORIS_DOWNLOAD="https://github.com/jemalloc/jemalloc/releases/download/5.3.0/jemalloc-5.3.0.tar.bz2"
Expand DownExpand Up@@ -398,11 +411,11 @@ BENCHMARK_SOURCE=benchmark-1.8.0
BENCHMARK_MD5SUM="8ddf8571d3f6198d37852bcbd964f817"

# xsimd
# for arrow-7.0.0, if arrow upgrade, this version may also need to be changed
XSIMD_DOWNLOAD="https://github.com/xtensor-stack/xsimd/archive/aeec9c872c8b475dedd7781336710f2dd2666cb2.tar.gz"
XSIMD_NAME=xsimd-aeec9c872c8b475dedd7781336710f2dd2666cb2.tar.gz
XSIMD_SOURCE=xsimd-aeec9c872c8b475dedd7781336710f2dd2666cb2
XSIMD_MD5SUM="d024855f71c0a2837a6918c0f8f66245"
# for arrow-13.0.0, if arrow upgrade, this version may also need to be changed
XSIMD_DOWNLOAD="https://github.com/xtensor-stack/xsimd/archive/refs/tags/9.0.1.tar.gz"
XSIMD_NAME="xsimd-9.0.1.tar.gz"
XSIMD_SOURCE=xsimd-9.0.1
XSIMD_MD5SUM="59f38fe3364acd7ed137771258812d6c"

# simdjson
SIMDJSON_DOWNLOAD="https://github.com/simdjson/simdjson/archive/refs/tags/v3.0.1.tar.gz"
Expand DownExpand Up@@ -505,6 +518,8 @@ export TP_ARCHIVES=(
 'CYRUS_SASL'
 'LIBRDKAFKA'
 'FLATBUFFERS'
 'CARES'
 'GRPC'
 'ARROW'
 'BROTLI'
 'ZSTD'
Expand Down