diff --git a/.dockerignore b/.dockerignore index 083905c74396..eb71138c679a 100644 --- a/.dockerignore +++ b/.dockerignore @@ -55,6 +55,8 @@ !rust/arrow-flight/Cargo.toml !rust/parquet/Cargo.toml !rust/parquet/build.rs +!rust/parquet_derive/Cargo.toml +!rust/parquet_derive_test/Cargo.toml !rust/datafusion/Cargo.toml !rust/datafusion/benches !rust/integration-testing/Cargo.toml diff --git a/.env b/.env index 62f5ae52b711..c266213d53c2 100644 --- a/.env +++ b/.env @@ -30,7 +30,7 @@ LLVM=10 CLANG_TOOLS=8 RUST=nightly-2020-04-22 GO=1.12 -NODE=11 +NODE=14 MAVEN=3.5.4 JDK=8 PANDAS=latest diff --git a/.github/workflows/archery.yml b/.github/workflows/archery.yml index 4eda469f49fa..b230bce82e1e 100644 --- a/.github/workflows/archery.yml +++ b/.github/workflows/archery.yml @@ -42,9 +42,10 @@ jobs: uses: actions/checkout@v2 with: fetch-depth: 0 - - name: Fetch Submodules and Tags + - name: Git Fixup + if: ${{ github.event_name == 'pull_request' }} shell: bash - run: ci/scripts/util_checkout.sh + run: git branch master origin/master - name: Free Up Disk Space run: ci/scripts/util_cleanup.sh - name: Setup Python diff --git a/.github/workflows/comment_bot.yml b/.github/workflows/comment_bot.yml index 340d6bc47191..b55da96beb2c 100644 --- a/.github/workflows/comment_bot.yml +++ b/.github/workflows/comment_bot.yml @@ -69,7 +69,7 @@ jobs: git remote add upstream https://github.com/apache/arrow git fetch upstream changed() { - git diff --name-only HEAD^..upstream/master | grep -e "$1" >/dev/null 2>&1 + git diff --name-only HEAD..upstream/master | grep -e "$1" >/dev/null 2>&1 } if changed '^r/.*\.R$'; then echo "::set-env name=R_DOCS::true" @@ -84,10 +84,11 @@ jobs: echo "::set-env name=CLANG_FORMAT_R::true" fi - name: Run cmake_format - if: false - # TODO: make this work https://issues.apache.org/jira/browse/ARROW-8489 - # if: env.CMAKE_FORMAT == 'true' || endsWith(github.event.comment.body, 'everything') + if: env.CMAKE_FORMAT == 'true' || endsWith(github.event.comment.body, 'everything') run: | + set -ex + export PATH=/home/runner/.local/bin:$PATH + python3 -m pip install --upgrade pip setuptools wheel python3 -m pip install -r dev/archery/requirements-lint.txt python3 run-cmake-format.py - name: Run clang-format on cpp diff --git a/.github/workflows/cpp.yml b/.github/workflows/cpp.yml index 716824ec9435..5ed1605a2a87 100644 --- a/.github/workflows/cpp.yml +++ b/.github/workflows/cpp.yml @@ -102,9 +102,11 @@ jobs: # hosted machines name: ${{ matrix.title }} runs-on: ${{ matrix.runner }} - if: github.event_name == 'push' + # TODO(kszucs): re-enable once the self-hosted workers are properly + # registered to github + if: false && github.event_name == 'push' defaults: - # to use certain environment variables are set by .bashrc an interactive + # To use certain environment variables set by .bashrc, an interactive # bash shell must be used run: shell: bash -i {0} @@ -119,11 +121,12 @@ jobs: debian: 10 title: ARM32v7 Debian 10 C++ image: | - -e CPP_MAKE_PARALLELISM=4 \ + -e CPP_MAKE_PARALLELISM=2 \ -e CXXFLAGS=-Wno-psabi \ -e ARROW_PARQUET=OFF \ -e ARROW_FLIGHT=OFF \ -e ARROW_GANDIVA=OFF \ + -e ARROW_ORC=OFF \ -e CMAKE_ARGS=-DARROW_CPU_FLAG=armv7 \ debian-cpp arch: 'arm32v7' @@ -174,15 +177,10 @@ jobs: uses: actions/checkout@v2 with: fetch-depth: 0 - - name: Build Docker image + - name: Run run: | cd cpp/examples/minimal_build - docker build -t arrow_cpp_minimal . - - name: Compile and run example - run: | - ARROW_ROOT=$PWD - cd cpp/examples/minimal_build - docker run -v $PWD:/io -v $ARROW_ROOT:/arrow arrow_cpp_minimal /io/run.sh + docker-compose run --rm minimal macos: name: AMD64 MacOS 10.15 C++ @@ -191,23 +189,24 @@ jobs: strategy: fail-fast: false env: + ARROW_BUILD_TESTS: ON + ARROW_DATASET: ON + ARROW_FLIGHT: ON + ARROW_GANDIVA: ON + ARROW_HDFS: ON ARROW_HOME: /usr/local ARROW_JEMALLOC: ON # TODO(kszucs): link error in the tests - ARROW_DATASET: ON ARROW_ORC: OFF - ARROW_FLIGHT: ON - ARROW_HDFS: ON - ARROW_PLASMA: ON - ARROW_GANDIVA: ON ARROW_PARQUET: ON - ARROW_WITH_ZLIB: ON - ARROW_WITH_LZ4: ON + ARROW_PLASMA: ON + ARROW_S3: ON + ARROW_WITH_BROTLI: ON ARROW_WITH_BZ2: ON - ARROW_WITH_ZSTD: ON + ARROW_WITH_LZ4: ON ARROW_WITH_SNAPPY: ON - ARROW_WITH_BROTLI: ON - ARROW_BUILD_TESTS: ON + ARROW_WITH_ZLIB: ON + ARROW_WITH_ZSTD: ON steps: - name: Checkout Arrow uses: actions/checkout@v2 @@ -324,6 +323,7 @@ jobs: ARROW_JEMALLOC: OFF ARROW_PARQUET: ON ARROW_PYTHON: ON + ARROW_S3: ON ARROW_USE_GLOG: OFF ARROW_VERBOSE_THIRDPARTY_BUILD: OFF ARROW_WITH_BROTLI: ON @@ -375,6 +375,14 @@ jobs: run: | export CMAKE_BUILD_PARALLEL_LEVEL=$NUMBER_OF_PROCESSORS ci/scripts/cpp_build.sh "$(pwd)" "$(pwd)/build" + - name: Download MinIO + shell: msys2 {0} + run: | + mkdir -p /usr/local/bin + wget \ + --output-document /usr/local/bin/minio.exe \ + https://dl.min.io/server/minio/release/windows-amd64/minio.exe + chmod +x /usr/local/bin/minio.exe - name: Test shell: msys2 {0} run: | diff --git a/.github/workflows/dev.yml b/.github/workflows/dev.yml index 59e820217227..c7f5821394df 100644 --- a/.github/workflows/dev.yml +++ b/.github/workflows/dev.yml @@ -59,43 +59,6 @@ jobs: continue-on-error: true run: archery docker push ubuntu-lint - docs: - name: Sphinx and API documentations - runs-on: ubuntu-latest - if: github.event_name == 'push' - steps: - - name: Checkout Arrow - uses: actions/checkout@v2 - with: - fetch-depth: 0 - - name: Fetch Submodules and Tags - shell: bash - run: ci/scripts/util_checkout.sh - - name: Free Up Disk Space - shell: bash - run: ci/scripts/util_cleanup.sh - - name: Cache Docker Volumes - uses: actions/cache@v1 - with: - path: .docker - key: ubuntu-18.04-${{ hashFiles('cpp/**') }} - restore-keys: ubuntu-18.04- - - name: Setup Python - uses: actions/setup-python@v1 - with: - python-version: 3.8 - - name: Setup Archery - run: pip install -e dev/archery[docker] - - name: Execute Docker Build - run: | - sudo sysctl -w kernel.core_pattern="core.%e.%p" - ulimit -c unlimited - archery docker run ubuntu-docs - - name: Docker Push - if: success() && github.event_name == 'push' && github.repository == 'apache/arrow' - continue-on-error: true - run: archery docker push ubuntu-docs - release: name: Source Release and Merge Script runs-on: ubuntu-latest diff --git a/.github/workflows/dev_labeler.yml b/.github/workflows/dev_labeler.yml new file mode 100644 index 000000000000..f5c3a3485672 --- /dev/null +++ b/.github/workflows/dev_labeler.yml @@ -0,0 +1,34 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: PR labeler +on: + pull_request_target: + types: [opened, reopened] + paths: + - 'rust/**' + +jobs: + assign-rust-labels: + runs-on: ubuntu-latest + steps: + - name: Assign Github labels + uses: actions/labeler@2.2.0 + with: + repo-token: ${{ secrets.GITHUB_TOKEN }} + configuration-path: .github/workflows/dev_labeler/labeler.yml + sync-labels: true diff --git a/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-eoan-arm64/from b/.github/workflows/dev_labeler/labeler.yml similarity index 92% rename from dev/tasks/linux-packages/apache-arrow/apt/ubuntu-eoan-arm64/from rename to .github/workflows/dev_labeler/labeler.yml index 7c4458fc9776..1753caa733f1 100644 --- a/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-eoan-arm64/from +++ b/.github/workflows/dev_labeler/labeler.yml @@ -15,4 +15,8 @@ # specific language governing permissions and limitations # under the License. -arm64v8/ubuntu:eoan +lang-rust: + - rust/**/* + +datafusion: + - rust/datafusion/**/* diff --git a/.github/workflows/java.yml b/.github/workflows/java.yml index a6796225e402..385d02101204 100644 --- a/.github/workflows/java.yml +++ b/.github/workflows/java.yml @@ -83,7 +83,7 @@ jobs: - name: Docker Push if: success() && github.event_name == 'push' && github.repository == 'apache/arrow' continue-on-error: true - run: archery docker push debian-go + run: archery docker push debian-java macos: name: AMD64 MacOS 10.15 Java JDK ${{ matrix.jdk }} diff --git a/.github/workflows/js.yml b/.github/workflows/js.yml index b12c2290c61e..e84a812a0783 100644 --- a/.github/workflows/js.yml +++ b/.github/workflows/js.yml @@ -40,7 +40,7 @@ env: jobs: docker: - name: AMD64 Debian 10 NodeJS 11 + name: AMD64 Debian 10 NodeJS 14 runs-on: ubuntu-latest if: ${{ !contains(github.event.pull_request.title, 'WIP') }} steps: @@ -75,7 +75,7 @@ jobs: strategy: fail-fast: false matrix: - node: [11] + node: [14] steps: - name: Checkout Arrow uses: actions/checkout@v2 @@ -103,7 +103,7 @@ jobs: # strategy: # fail-fast: false # matrix: - # node: [11] + # node: [14] # steps: # - name: Checkout Arrow # uses: actions/checkout@v1 diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index 84c5bb601ee4..c0582b4ee643 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -116,7 +116,7 @@ jobs: run: archery docker push ${{ matrix.image }} macos: - name: AMD64 MacOS 10.15 Python 3.7 + name: AMD64 MacOS 10.15 Python 3 runs-on: macos-latest if: ${{ !contains(github.event.pull_request.title, 'WIP') }} env: @@ -150,8 +150,9 @@ jobs: brew update --preinstall brew bundle --file=cpp/Brewfile brew install coreutils python - pip3 install -r python/requirements-build.txt \ - -r python/requirements-test.txt + python3 -mpip install \ + -r python/requirements-build.txt \ + -r python/requirements-test.txt - name: Build shell: bash run: | diff --git a/.github/workflows/r.yml b/.github/workflows/r.yml index 29ffd4443707..6f782c220631 100644 --- a/.github/workflows/r.yml +++ b/.github/workflows/r.yml @@ -53,7 +53,7 @@ jobs: strategy: fail-fast: false matrix: - r: ["3.6", "4.0"] + r: ["3.6"] ubuntu: [18.04] env: R: ${{ matrix.r }} @@ -100,7 +100,7 @@ jobs: fail-fast: false matrix: # See https://hub.docker.com/r/rstudio/r-base - r_version: ["3.6", "4.0"] + r_version: ["4.0"] r_image: - centos7 env: diff --git a/.github/workflows/ruby.yml b/.github/workflows/ruby.yml index 783ca91cb825..d7c81bc76265 100644 --- a/.github/workflows/ruby.yml +++ b/.github/workflows/ruby.yml @@ -174,6 +174,7 @@ jobs: ARROW_JEMALLOC: OFF ARROW_PARQUET: ON ARROW_PYTHON: OFF + ARROW_S3: ON ARROW_USE_GLOG: OFF ARROW_WITH_BROTLI: ON ARROW_WITH_BZ2: ON diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 89724a0a412f..e1ba59987f22 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -152,12 +152,8 @@ jobs: - name: Fetch Submodules and Tags shell: bash run: ci/scripts/util_checkout.sh - - name: Cache Build Artifacts - uses: actions/cache@v1 - with: - path: rust/target - key: macos-10-rust-${{ hashFiles('rust/**.rs') }} - restore-keys: macos-10-rust- + # Note we don't cache the build artifacts for MacOSX as it was causing intermittent failures + # see https://issues.apache.org/jira/browse/ARROW-9628 - name: Run Clippy shell: bash run: ci/scripts/rust_lint.sh $(pwd) diff --git a/.github/workflows/rust_cron.yml b/.github/workflows/rust_cron.yml new file mode 100644 index 000000000000..378f2dd10811 --- /dev/null +++ b/.github/workflows/rust_cron.yml @@ -0,0 +1,56 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: Rust Cron + +on: + push: + paths: + - '.github/workflows/rust_cron.yml' + pull_request: + paths: + - '.github/workflows/rust_cron.yml' + schedule: + - cron: 0 */12 * * * + +jobs: + coverage: + name: AMD64 Debian 10 Rust ${{ matrix.rust }} Coverage + runs-on: ubuntu-latest + if: ${{ !contains(github.event.pull_request.title, 'WIP') && github.repository == 'apache/arrow' }} + strategy: + fail-fast: false + matrix: + rust: [nightly-2020-04-22] + env: + RUST: ${{ matrix.rust }} + steps: + - name: Checkout Arrow + uses: actions/checkout@v2 + with: + fetch-depth: 0 + - name: Fetch Submodules and Tags + run: ci/scripts/util_checkout.sh + - name: Run coverage + shell: bash + run: | + echo ${RUST} > rust/rust-toolchain && + ci/scripts/rust_coverage.sh `pwd` `pwd`/build $RUST + - name: Report coverage + continue-on-error: true + shell: bash + run: bash <(curl -s https://codecov.io/bash) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index d72b126ef34e..e70eaceaf417 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -55,7 +55,7 @@ repos: hooks: - id: flake8 name: Python Format - files: ^(python|crossbow|integration)/ + files: ^(python|dev|integration)/ types: - file - python diff --git a/.travis.yml b/.travis.yml index fa5d84c82c7c..72eed9bd4109 100644 --- a/.travis.yml +++ b/.travis.yml @@ -40,22 +40,75 @@ jobs: ARCH: arm64v8 ARROW_CI_MODULES: "CPP" DOCKER_IMAGE_ID: ubuntu-cpp + # ARROW_USE_GLOG=OFF is needed to avoid build error caused by + # glog and CMAKE_UNITY_BUILD=ON. + # + # Disable ARROW_S3 because it often causes "No output has + # been received in the last 10m0s, this potentially indicates + # a stalled build or something wrong with the build itself." + # on Travis CI. + DOCKER_RUN_ARGS: >- + " + -e ARROW_BUILD_STATIC=OFF + -e ARROW_ORC=OFF + -e ARROW_S3=OFF + -e ARROW_USE_GLOG=OFF + -e CMAKE_UNITY_BUILD=ON + " + # We need to use smaller build when cache doesn't exist + # because Travis CI has "No output has been received in the + # last 10m0s" limitation. If we build many modules, we reach + # the limitation. + DOCKER_RUN_ARGS_NO_CACHE: >- + " + -e ARROW_BUILD_TESTS=OFF + -e ARROW_GANDIVA=OFF + -e ARROW_PARQUET=OFF + " UBUNTU: "20.04" + - name: "C++ on s390x" os: linux arch: s390x env: ARCH: s390x ARROW_CI_MODULES: "CPP" - ARROW_FLIGHT: "ON" - ARROW_PARQUET: "OFF" DOCKER_IMAGE_ID: ubuntu-cpp - PARQUET_BUILD_EXAMPLES: "OFF" - PARQUET_BUILD_EXECUTABLES: "OFF" - Protobuf_SOURCE: "BUNDLED" + # Can't use CMAKE_UNITIFY_BUILD=ON because of compiler crash. + # Can't enable ARROW_S3 because compiler is killed while compiling + # aws-sdk-cpp. + DOCKER_RUN_ARGS: >- + " + -e ARROW_BUILD_STATIC=OFF + -e ARROW_FLIGHT=ON + -e ARROW_ORC=OFF + -e ARROW_PARQUET=OFF + -e ARROW_S3=OFF + -e PARQUET_BUILD_EXAMPLES=OFF + -e PARQUET_BUILD_EXECUTABLES=OFF + -e Protobuf_SOURCE=BUNDLED + -e cares_SOURCE=BUNDLED + -e gRPC_SOURCE=BUNDLED + " UBUNTU: "20.04" - cares_SOURCE: "BUNDLED" - gRPC_SOURCE: "BUNDLED" + + - name: "Go on s390x" + os: linux + arch: s390x + env: + ARCH: s390x + ARROW_CI_MODULES: "GO" + DOCKER_IMAGE_ID: debian-go + + - name: "Java on s390x" + os: linux + arch: s390x + env: + ARCH: s390x + ARROW_CI_MODULES: "JAVA" + DOCKER_IMAGE_ID: debian-java + JDK: 11 + allow_failures: - arch: s390x @@ -87,15 +140,13 @@ script: # /home/travis/.travis/functions: line 109: ulimit: core file size: cannot modify limit: Operation not permitted - | ulimit -c unlimited || : + - | + if [ $(ls $TRAVIS_BUILD_DIR/.docker | wc -l) -eq 0 ]; then + DOCKER_RUN_ARGS="${DOCKER_RUN_ARGS} ${DOCKER_RUN_ARGS_NO_CACHE}" + fi - | archery docker run \ - -e ARROW_FLIGHT=${ARROW_FLIGHT:-OFF} \ - -e ARROW_PARQUET=${ARROW_PARQUET:-ON} \ - -e PARQUET_BUILD_EXAMPLES=${PARQUET_BUILD_EXAMPLES:-ON} \ - -e PARQUET_BUILD_EXECUTABLES=${PARQUET_BUILD_EXECUTABLES:-ON} \ - -e Protobuf_SOURCE=${Protobuf_SOURCE:-} \ - -e cares_SOURCE=${cares_SOURCE:-} \ - -e gRPC_SOURCE=${gRPC_SOURCE:-} \ + ${DOCKER_RUN_ARGS} \ --volume ${PWD}/build:/build \ ${DOCKER_IMAGE_ID} diff --git a/CHANGELOG.md b/CHANGELOG.md index 3e5619b9a33d..686f1d4ef5e1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,588 @@ +# Apache Arrow 2.0.0 (2020-10-13) + +## Bug Fixes + +* [ARROW-2367](https://issues.apache.org/jira/browse/ARROW-2367) - [Python] ListArray has trouble with sizes greater than kMaximumCapacity +* [ARROW-4189](https://issues.apache.org/jira/browse/ARROW-4189) - [CI] [Rust] Fix broken cargo coverage +* [ARROW-4917](https://issues.apache.org/jira/browse/ARROW-4917) - [C++] orc\_ep fails in cpp-alpine docker +* [ARROW-5578](https://issues.apache.org/jira/browse/ARROW-5578) - [C++][Flight] Flight does not build out of the box on Alpine Linux +* [ARROW-7226](https://issues.apache.org/jira/browse/ARROW-7226) - [JSON][Python] Json loader fails on example in documentation. +* [ARROW-7384](https://issues.apache.org/jira/browse/ARROW-7384) - [Website] Fix search indexing warning reported by Google +* [ARROW-7517](https://issues.apache.org/jira/browse/ARROW-7517) - [C++] Builder does not honour dictionary type provided during initialization +* [ARROW-7663](https://issues.apache.org/jira/browse/ARROW-7663) - [Python] from\_pandas gives TypeError instead of ArrowTypeError in some cases +* [ARROW-7903](https://issues.apache.org/jira/browse/ARROW-7903) - [Rust] [DataFusion] Upgrade SQLParser dependency for DataFusion +* [ARROW-7957](https://issues.apache.org/jira/browse/ARROW-7957) - [Python] ParquetDataset cannot take HadoopFileSystem as filesystem +* [ARROW-8265](https://issues.apache.org/jira/browse/ARROW-8265) - [Rust] [DataFusion] Table API collect() should not require context +* [ARROW-8394](https://issues.apache.org/jira/browse/ARROW-8394) - [JS] Typescript compiler errors for arrow d.ts files, when using es2015-esm package +* [ARROW-8735](https://issues.apache.org/jira/browse/ARROW-8735) - [Rust] [Parquet] Parquet crate fails to compile on Arm architecture +* [ARROW-8749](https://issues.apache.org/jira/browse/ARROW-8749) - [C++] IpcFormatWriter writes dictionary batches with wrong ID +* [ARROW-8773](https://issues.apache.org/jira/browse/ARROW-8773) - [Python] pyarrow schema.empty\_table() does not preserve nullability of fields +* [ARROW-9028](https://issues.apache.org/jira/browse/ARROW-9028) - [R] Should be able to convert an empty table +* [ARROW-9096](https://issues.apache.org/jira/browse/ARROW-9096) - [Python] Pandas roundtrip with object-dtype column labels with integer values: data type "integer" not understood +* [ARROW-9177](https://issues.apache.org/jira/browse/ARROW-9177) - [C++][Parquet] Tracking issue for cross-implementation LZ4 Parquet compression compatibility +* [ARROW-9414](https://issues.apache.org/jira/browse/ARROW-9414) - [C++] apt package includes headers for S3 interface, but no support +* [ARROW-9462](https://issues.apache.org/jira/browse/ARROW-9462) - [Go] The Indentation after the first Record arrjson writer is missing +* [ARROW-9463](https://issues.apache.org/jira/browse/ARROW-9463) - [Go] The writer is double closed in TestReadWrite +* [ARROW-9490](https://issues.apache.org/jira/browse/ARROW-9490) - [Python] pyarrow array creation for specific set of numpy scalars fails +* [ARROW-9495](https://issues.apache.org/jira/browse/ARROW-9495) - [C++] Equality assertions don't handle Inf / -Inf properly +* [ARROW-9520](https://issues.apache.org/jira/browse/ARROW-9520) - [Rust] [DataFusion] Can't alias an aggregate expression +* [ARROW-9528](https://issues.apache.org/jira/browse/ARROW-9528) - [Python] Honor tzinfo information when converting from datetime to pyarrow +* [ARROW-9532](https://issues.apache.org/jira/browse/ARROW-9532) - [Python] Building pyarrow for MacPorts on macOS +* [ARROW-9535](https://issues.apache.org/jira/browse/ARROW-9535) - [Python] Remove symlink fixes from conda recipe +* [ARROW-9536](https://issues.apache.org/jira/browse/ARROW-9536) - Missing parameters in PlasmaOutOfMemoryException.java +* [ARROW-9541](https://issues.apache.org/jira/browse/ARROW-9541) - [C++] CMakeLists requires UTF8PROC\_STATIC when building static library +* [ARROW-9544](https://issues.apache.org/jira/browse/ARROW-9544) - [R] version argument of write\_parquet not working +* [ARROW-9546](https://issues.apache.org/jira/browse/ARROW-9546) - [Python] Clean up Pandas Metadata Conversion test +* [ARROW-9548](https://issues.apache.org/jira/browse/ARROW-9548) - [Go] Test output files in tmp directory are not removed correctly +* [ARROW-9549](https://issues.apache.org/jira/browse/ARROW-9549) - [Rust] Parquet no longer builds +* [ARROW-9554](https://issues.apache.org/jira/browse/ARROW-9554) - [Java] FixedWidthInPlaceVectorSorter sometimes produces wrong result +* [ARROW-9556](https://issues.apache.org/jira/browse/ARROW-9556) - [Python][C++] Segfaults in UnionArray with null values +* [ARROW-9560](https://issues.apache.org/jira/browse/ARROW-9560) - [Packaging] conda recipes failing due to missing conda-forge.yml +* [ARROW-9569](https://issues.apache.org/jira/browse/ARROW-9569) - [CI][R] Fix rtools35 builds for msys2 key change +* [ARROW-9570](https://issues.apache.org/jira/browse/ARROW-9570) - [Doc] Clean up sphinx sidebar +* [ARROW-9573](https://issues.apache.org/jira/browse/ARROW-9573) - [Python] Parquet doesn't load when partitioned column starts with '\_' +* [ARROW-9574](https://issues.apache.org/jira/browse/ARROW-9574) - [R] Cleanups for CRAN 1.0.0 release +* [ARROW-9575](https://issues.apache.org/jira/browse/ARROW-9575) - [R] gcc-UBSAN failure on CRAN +* [ARROW-9577](https://issues.apache.org/jira/browse/ARROW-9577) - [Python][C++] posix\_madvise error on Debian in pyarrow 1.0.0 +* [ARROW-9583](https://issues.apache.org/jira/browse/ARROW-9583) - [Rust] Offset is mishandled in arithmetic and boolean compute kernels +* [ARROW-9588](https://issues.apache.org/jira/browse/ARROW-9588) - [C++] clang/win: Copy constructor of ParquetInvalidOrCorruptedFileException not correctly triggered +* [ARROW-9589](https://issues.apache.org/jira/browse/ARROW-9589) - [C++/R] arrow\_exports.h contains structs declared as class +* [ARROW-9592](https://issues.apache.org/jira/browse/ARROW-9592) - [CI] Update homebrew before calling brew bundle +* [ARROW-9596](https://issues.apache.org/jira/browse/ARROW-9596) - [CI][Crossbow] Fix homebrew-cpp again, again +* [ARROW-9597](https://issues.apache.org/jira/browse/ARROW-9597) - [C++] AddAlias in compute::FunctionRegistry should be synchronized +* [ARROW-9598](https://issues.apache.org/jira/browse/ARROW-9598) - [C++][Parquet] Spaced definition levels is not assigned correctly. +* [ARROW-9599](https://issues.apache.org/jira/browse/ARROW-9599) - [CI] Appveyor toolchain build fails because CMake detects different C and C++ compilers +* [ARROW-9600](https://issues.apache.org/jira/browse/ARROW-9600) - [Rust] When used as a crate dependency, arrow-flight is rebuilt on every invocation of cargo build +* [ARROW-9600](https://issues.apache.org/jira/browse/ARROW-9600) - [Rust] When used as a crate dependency, arrow-flight is rebuilt on every invocation of cargo build +* [ARROW-9602](https://issues.apache.org/jira/browse/ARROW-9602) - [R] Improve cmake detection in Linux build +* [ARROW-9603](https://issues.apache.org/jira/browse/ARROW-9603) - [C++][Parquet] Write Arrow relies on unspecified behavior for nested types +* [ARROW-9606](https://issues.apache.org/jira/browse/ARROW-9606) - [C++][Dataset] in expressions don't work with \>1 partition levels +* [ARROW-9609](https://issues.apache.org/jira/browse/ARROW-9609) - [C++] CSV datasets don't materialize virtual columns +* [ARROW-9621](https://issues.apache.org/jira/browse/ARROW-9621) - [Python] test\_move\_file() is failed with fsspec 0.8.0 +* [ARROW-9622](https://issues.apache.org/jira/browse/ARROW-9622) - [Java] ComplexCopier fails if a structvector has a child UnionVector with nulls +* [ARROW-9628](https://issues.apache.org/jira/browse/ARROW-9628) - [Rust] Clippy PR test failing intermittently on Rust / AMD64 MacOS +* [ARROW-9629](https://issues.apache.org/jira/browse/ARROW-9629) - [Python] Kartothek integration tests failing due to missing freezegun module +* [ARROW-9631](https://issues.apache.org/jira/browse/ARROW-9631) - [Rust] Arrow crate should not depend on flight +* [ARROW-9631](https://issues.apache.org/jira/browse/ARROW-9631) - [Rust] Arrow crate should not depend on flight +* [ARROW-9642](https://issues.apache.org/jira/browse/ARROW-9642) - [C++] Let MakeBuilder refer DictionaryType's index\_type for deciding the starting bit width of the indices +* [ARROW-9643](https://issues.apache.org/jira/browse/ARROW-9643) - [C++] Illegal instruction on haswell cpu +* [ARROW-9644](https://issues.apache.org/jira/browse/ARROW-9644) - [C++][Dataset] Do not check for ignore\_prefixes in the base path +* [ARROW-9652](https://issues.apache.org/jira/browse/ARROW-9652) - [Rust][DataFusion] Panic trying to select \* from a CSV (panicked at 'index out of bounds: the len is 0 but the index is 0) +* [ARROW-9653](https://issues.apache.org/jira/browse/ARROW-9653) - [Rust][DataFusion] Multi-column Group by: Invalid Argument Error +* [ARROW-9659](https://issues.apache.org/jira/browse/ARROW-9659) - [C++] RecordBatchStreamReader throws on CUDA device buffers +* [ARROW-9660](https://issues.apache.org/jira/browse/ARROW-9660) - [C++] IPC - dictionaries in maps +* [ARROW-9666](https://issues.apache.org/jira/browse/ARROW-9666) - [Python][wheel][Windows] library missing failure by ARROW-9412 +* [ARROW-9670](https://issues.apache.org/jira/browse/ARROW-9670) - [C++][FlightRPC] Close()ing a DoPut with an ongoing read locks up the client +* [ARROW-9684](https://issues.apache.org/jira/browse/ARROW-9684) - [C++] Fix undefined behaviour on invalid IPC / Parquet input (OSS-Fuzz) +* [ARROW-9692](https://issues.apache.org/jira/browse/ARROW-9692) - [Python] distutils import warning +* [ARROW-9693](https://issues.apache.org/jira/browse/ARROW-9693) - [CI][Docs] Nightly docs build fails +* [ARROW-9696](https://issues.apache.org/jira/browse/ARROW-9696) - [Rust] [Datafusion] nested binary expressions broken +* [ARROW-9698](https://issues.apache.org/jira/browse/ARROW-9698) - [C++] Revert "Add -NDEBUG flag to arrow.pc" +* [ARROW-9700](https://issues.apache.org/jira/browse/ARROW-9700) - [Python] create\_library\_symlinks doesn't work in macos +* [ARROW-9712](https://issues.apache.org/jira/browse/ARROW-9712) - [Rust] [DataFusion] ParquetScanExec panics on error +* [ARROW-9714](https://issues.apache.org/jira/browse/ARROW-9714) - [Rust] [DataFusion] TypeCoercionRule not implemented for Limit or Sort +* [ARROW-9716](https://issues.apache.org/jira/browse/ARROW-9716) - [Rust] [DataFusion] MergeExec should have concurrency limit +* [ARROW-9726](https://issues.apache.org/jira/browse/ARROW-9726) - [Rust] [DataFusion] ParquetScanExec launches threads too early +* [ARROW-9727](https://issues.apache.org/jira/browse/ARROW-9727) - [C++] Fix crash on invalid IPC input (OSS-Fuzz) +* [ARROW-9729](https://issues.apache.org/jira/browse/ARROW-9729) - [Java] Error Prone causes other annotation processors to not work with Eclipse +* [ARROW-9733](https://issues.apache.org/jira/browse/ARROW-9733) - [Rust][DataFusion] Aggregates COUNT/MIN/MAX don't work on VARCHAR columns +* [ARROW-9734](https://issues.apache.org/jira/browse/ARROW-9734) - [Rust] [DataFusion] TableProvider.scan executing partitions prematurely +* [ARROW-9741](https://issues.apache.org/jira/browse/ARROW-9741) - [Rust] [DataFusion] Incorrect count in TPC-H query 1 result set +* [ARROW-9743](https://issues.apache.org/jira/browse/ARROW-9743) - [R] Sanitize paths in open\_dataset +* [ARROW-9744](https://issues.apache.org/jira/browse/ARROW-9744) - [Python] Failed to install on aarch64 +* [ARROW-9764](https://issues.apache.org/jira/browse/ARROW-9764) - [CI][Java] Push wrong Docker image +* [ARROW-9768](https://issues.apache.org/jira/browse/ARROW-9768) - [Python] Pyarrow allows for unsafe conversions of datetime objects to timestamp nanoseconds +* [ARROW-9768](https://issues.apache.org/jira/browse/ARROW-9768) - [Python] Pyarrow allows for unsafe conversions of datetime objects to timestamp nanoseconds +* [ARROW-9778](https://issues.apache.org/jira/browse/ARROW-9778) - [Rust] [DataFusion] Logical and physical schemas' nullability does not match in 8 out of 20 end-to-end tests +* [ARROW-9783](https://issues.apache.org/jira/browse/ARROW-9783) - [Rust] [DataFusion] Logical aggregate expressions require explicit data type +* [ARROW-9785](https://issues.apache.org/jira/browse/ARROW-9785) - [Python] pyarrow/tests/test\_fs.py::test\_s3\_options too slow +* [ARROW-9789](https://issues.apache.org/jira/browse/ARROW-9789) - [C++] Don't install jemalloc in parallel +* [ARROW-9790](https://issues.apache.org/jira/browse/ARROW-9790) - [Rust] [Parquet] ParquetFileArrowReader fails to decode all pages if batches fall exactly on row group boundaries +* [ARROW-9790](https://issues.apache.org/jira/browse/ARROW-9790) - [Rust] [Parquet] ParquetFileArrowReader fails to decode all pages if batches fall exactly on row group boundaries +* [ARROW-9793](https://issues.apache.org/jira/browse/ARROW-9793) - [Rust] [DataFusion] Tests failing in master +* [ARROW-9797](https://issues.apache.org/jira/browse/ARROW-9797) - [Rust] AMD64 Conda Integration Tests is failing for the Master branch +* [ARROW-9799](https://issues.apache.org/jira/browse/ARROW-9799) - [Rust] [DataFusion] Implementation of physical binary expression get\_type method is incorrect +* [ARROW-9800](https://issues.apache.org/jira/browse/ARROW-9800) - [Rust] [Parquet] "min" and "max" written to standard out when writing columns +* [ARROW-9809](https://issues.apache.org/jira/browse/ARROW-9809) - [Rust] [DataFusion] logical schema = physical schema is not true +* [ARROW-9814](https://issues.apache.org/jira/browse/ARROW-9814) - [Python] Crash in test\_parquet.py::test\_read\_partitioned\_directory\_s3fs +* [ARROW-9815](https://issues.apache.org/jira/browse/ARROW-9815) - [Rust] [DataFusion] Deadlock in creation of physical plan with two udfs +* [ARROW-9815](https://issues.apache.org/jira/browse/ARROW-9815) - [Rust] [DataFusion] Deadlock in creation of physical plan with two udfs +* [ARROW-9815](https://issues.apache.org/jira/browse/ARROW-9815) - [Rust] [DataFusion] Deadlock in creation of physical plan with two udfs +* [ARROW-9816](https://issues.apache.org/jira/browse/ARROW-9816) - [C++] Escape quotes in config.h +* [ARROW-9827](https://issues.apache.org/jira/browse/ARROW-9827) - [Python] pandas.read\_parquet fails for wide parquet files and pyarrow 1.0.X +* [ARROW-9831](https://issues.apache.org/jira/browse/ARROW-9831) - [Rust] [DataFusion] Fix compilation error +* [ARROW-9840](https://issues.apache.org/jira/browse/ARROW-9840) - [Python] Python fs documentation out of date with code +* [ARROW-9846](https://issues.apache.org/jira/browse/ARROW-9846) - [Rust] Master branch broken build +* [ARROW-9851](https://issues.apache.org/jira/browse/ARROW-9851) - [C++] Valgrind errors due to unrecognized instructions +* [ARROW-9852](https://issues.apache.org/jira/browse/ARROW-9852) - [C++] Fix crash on invalid IPC input (OSS-Fuzz) +* [ARROW-9852](https://issues.apache.org/jira/browse/ARROW-9852) - [C++] Fix crash on invalid IPC input (OSS-Fuzz) +* [ARROW-9855](https://issues.apache.org/jira/browse/ARROW-9855) - [R] Fix bad merge/Rcpp conflict +* [ARROW-9859](https://issues.apache.org/jira/browse/ARROW-9859) - [C++] S3 FileSystemFromUri with special char in secret key fails +* [ARROW-9864](https://issues.apache.org/jira/browse/ARROW-9864) - [Python] pathlib.Path not supported in write\_to\_dataset with partition columns +* [ARROW-9874](https://issues.apache.org/jira/browse/ARROW-9874) - [C++] NewStreamWriter / NewFileWriter don't own output stream +* [ARROW-9876](https://issues.apache.org/jira/browse/ARROW-9876) - [CI][C++] Travis ARM jobs timeout +* [ARROW-9877](https://issues.apache.org/jira/browse/ARROW-9877) - [C++][CI] homebrew-cpp fails due to avx512 +* [ARROW-9879](https://issues.apache.org/jira/browse/ARROW-9879) - [Python] ChunkedArray.\_\_getitem\_\_ doesn't work with numpy scalars +* [ARROW-9882](https://issues.apache.org/jira/browse/ARROW-9882) - [C++/Python] Update conda-forge-pinning to 3 for OSX conda packages +* [ARROW-9883](https://issues.apache.org/jira/browse/ARROW-9883) - [R] Fix linuxlibs.R install script for R < 3.6 +* [ARROW-9888](https://issues.apache.org/jira/browse/ARROW-9888) - [Rust] [DataFusion] ExecutionContext can not be shared between threads +* [ARROW-9889](https://issues.apache.org/jira/browse/ARROW-9889) - [Rust][DataFusion] Datafusion CLI: CREATE EXTERNAL TABLE errors with "Unsupported logical plan variant" +* [ARROW-9897](https://issues.apache.org/jira/browse/ARROW-9897) - [C++][Gandiva] Add to\_date() function from pattern +* [ARROW-9906](https://issues.apache.org/jira/browse/ARROW-9906) - [Python] Crash in test\_parquet.py::test\_parquet\_writer\_filesystem\_s3\_uri (closing NativeFile from S3FileSystem) +* [ARROW-9913](https://issues.apache.org/jira/browse/ARROW-9913) - [C++] Outputs of Decimal128::FromString depend on presence of one another +* [ARROW-9920](https://issues.apache.org/jira/browse/ARROW-9920) - [Python] pyarrow.concat\_arrays segfaults when passing it a chunked array +* [ARROW-9922](https://issues.apache.org/jira/browse/ARROW-9922) - [Rust] Add \`try\_from(Vec\>)\` to StructArray +* [ARROW-9924](https://issues.apache.org/jira/browse/ARROW-9924) - [Python] Performance regression reading individual Parquet files using Dataset interface +* [ARROW-9931](https://issues.apache.org/jira/browse/ARROW-9931) - [C++] Fix undefined behaviour on invalid IPC (OSS-Fuzz) +* [ARROW-9932](https://issues.apache.org/jira/browse/ARROW-9932) - [R] Arrow 1.0.1 R package fails to install on R3.4 over linux +* [ARROW-9936](https://issues.apache.org/jira/browse/ARROW-9936) - [Python] Fix / test relative file paths in pyarrow.parquet +* [ARROW-9937](https://issues.apache.org/jira/browse/ARROW-9937) - [Rust] [DataFusion] Average is not correct +* [ARROW-9943](https://issues.apache.org/jira/browse/ARROW-9943) - [C++] Arrow metadata not applied recursively when reading Parquet file +* [ARROW-9946](https://issues.apache.org/jira/browse/ARROW-9946) - [R] ParquetFileWriter segfaults when \`sink\` is a string +* [ARROW-9953](https://issues.apache.org/jira/browse/ARROW-9953) - [R] Declare minimum version for bit64 +* [ARROW-9962](https://issues.apache.org/jira/browse/ARROW-9962) - [Python] Conversion to pandas with index column using fixed timezone fails +* [ARROW-9968](https://issues.apache.org/jira/browse/ARROW-9968) - [C++] UBSAN link failure with \_\_int8\_t +* [ARROW-9969](https://issues.apache.org/jira/browse/ARROW-9969) - [C++] RecordBatchBuilder yields invalid result with dictionary fields +* [ARROW-9970](https://issues.apache.org/jira/browse/ARROW-9970) - [Go] checkptr failures in sum methods +* [ARROW-9972](https://issues.apache.org/jira/browse/ARROW-9972) - [CI] Work around grpc-re2 clash on Homebrew +* [ARROW-9973](https://issues.apache.org/jira/browse/ARROW-9973) - [Java] JDBC DateConsumer does not allow dates before epoch +* [ARROW-9976](https://issues.apache.org/jira/browse/ARROW-9976) - [Python] ArrowCapacityError when doing Table.from\_pandas with large dataframe +* [ARROW-9990](https://issues.apache.org/jira/browse/ARROW-9990) - [Rust] [DataFusion] NOT is not plannable +* [ARROW-9993](https://issues.apache.org/jira/browse/ARROW-9993) - [Python] Tzinfo - string roundtrip fails on pytz.StaticTzInfo objects +* [ARROW-9994](https://issues.apache.org/jira/browse/ARROW-9994) - [C++][Python] Auto chunking nested array containing binary-like fields result malformed output +* [ARROW-9996](https://issues.apache.org/jira/browse/ARROW-9996) - [C++] Dictionary is unset when calling DictionaryArray.GetScalar for null values +* [ARROW-10003](https://issues.apache.org/jira/browse/ARROW-10003) - [C++] Create directories in CopyFiles when copying within the same filesystem +* [ARROW-10008](https://issues.apache.org/jira/browse/ARROW-10008) - [Python] pyarrow.parquet.read\_table fails with predicate pushdown on categorical data with use\_legacy\_dataset=False +* [ARROW-10011](https://issues.apache.org/jira/browse/ARROW-10011) - [C++] Make FindRE2.cmake re-entrant +* [ARROW-10012](https://issues.apache.org/jira/browse/ARROW-10012) - [C++] Sporadic failures in CopyFiles test +* [ARROW-10013](https://issues.apache.org/jira/browse/ARROW-10013) - [C++][CI] Flight test failure in TestFlightClient.GenericOptions +* [ARROW-10017](https://issues.apache.org/jira/browse/ARROW-10017) - [Java] LargeMemoryUtil.checkedCastToInt has buggy logic +* [ARROW-10022](https://issues.apache.org/jira/browse/ARROW-10022) - [C++] [Compute] core dumped on some scalar-arithmetic-benchmark +* [ARROW-10027](https://issues.apache.org/jira/browse/ARROW-10027) - [Python] Incorrect null column returned when using a dataset filter expression. +* [ARROW-10034](https://issues.apache.org/jira/browse/ARROW-10034) - [Rust] Master build broken +* [ARROW-10041](https://issues.apache.org/jira/browse/ARROW-10041) - [Rust] Possible to create LargeStringArray with DataType::Utf8 +* [ARROW-10047](https://issues.apache.org/jira/browse/ARROW-10047) - [CI] Conda integration tests failing with cmake error +* [ARROW-10048](https://issues.apache.org/jira/browse/ARROW-10048) - [Rust] Error in aggregate of min/max for strings +* [ARROW-10049](https://issues.apache.org/jira/browse/ARROW-10049) - [C++/Python] Sync conda recipe with conda-forge +* [ARROW-10060](https://issues.apache.org/jira/browse/ARROW-10060) - [Rust] [DataFusion] MergeExec currently discards partitions with errors +* [ARROW-10062](https://issues.apache.org/jira/browse/ARROW-10062) - [Rust]: Fix for null elems for DoubleEndedIter for DictArray +* [ARROW-10073](https://issues.apache.org/jira/browse/ARROW-10073) - [Python] Test test\_parquet\_nested\_storage relies on dict item ordering +* [ARROW-10081](https://issues.apache.org/jira/browse/ARROW-10081) - [C++/Python] Fix bash syntax in drone.io conda builds +* [ARROW-10085](https://issues.apache.org/jira/browse/ARROW-10085) - [C++] S3 tests fail on AppVeyor +* [ARROW-10087](https://issues.apache.org/jira/browse/ARROW-10087) - [CI] Fix nightly docs job +* [ARROW-10098](https://issues.apache.org/jira/browse/ARROW-10098) - [R][Doc] Fix copy\_files doc mismatch +* [ARROW-10104](https://issues.apache.org/jira/browse/ARROW-10104) - [Python] Separate tests into its own conda package +* [ARROW-10114](https://issues.apache.org/jira/browse/ARROW-10114) - [R] Segfault in to\_dataframe\_parallel with deeply nested structs +* [ARROW-10116](https://issues.apache.org/jira/browse/ARROW-10116) - [Python][Packaging] Fix gRPC linking error in macOS wheels builds +* [ARROW-10119](https://issues.apache.org/jira/browse/ARROW-10119) - [C++] Fix Parquet crashes on invalid input (OSS-Fuzz) +* [ARROW-10121](https://issues.apache.org/jira/browse/ARROW-10121) - [C++][Python] Variable dictionaries do not survive roundtrip to IPC stream +* [ARROW-10124](https://issues.apache.org/jira/browse/ARROW-10124) - [R] Write functions don't follow umask setting +* [ARROW-10125](https://issues.apache.org/jira/browse/ARROW-10125) - [R] Int64 downcast check doesn't consider all chunks +* [ARROW-10130](https://issues.apache.org/jira/browse/ARROW-10130) - [C++][Dataset] ParquetFileFragment::SplitByRowGroup does not preserve "complete\_metadata" status +* [ARROW-10136](https://issues.apache.org/jira/browse/ARROW-10136) - [Rust][Arrow] Nulls are transformed into "" after filtering for StringArray +* [ARROW-10137](https://issues.apache.org/jira/browse/ARROW-10137) - [R] Fix cpp helper that breaks if libarrow is not present +* [ARROW-10147](https://issues.apache.org/jira/browse/ARROW-10147) - [Python] Constructing pandas metadata fails if an Index name is not JSON-serializable by default +* [ARROW-10150](https://issues.apache.org/jira/browse/ARROW-10150) - [C++] Fix crashes on invalid Parquet file (OSS-Fuzz) +* [ARROW-10169](https://issues.apache.org/jira/browse/ARROW-10169) - [Rust] Nulls should be rendered as "" rather than default value when pretty printing arrays +* [ARROW-10175](https://issues.apache.org/jira/browse/ARROW-10175) - [CI] Nightly hdfs integration test job fails +* [ARROW-10176](https://issues.apache.org/jira/browse/ARROW-10176) - [CI] Nightly valgrind job fails +* [ARROW-10178](https://issues.apache.org/jira/browse/ARROW-10178) - [CI] Fix spark master integration test build setup +* [ARROW-10179](https://issues.apache.org/jira/browse/ARROW-10179) - [Rust] Labeler is not labeling +* [ARROW-10181](https://issues.apache.org/jira/browse/ARROW-10181) - [Rust] Arrow tests fail to compile on Raspberry Pi (32 bit) +* [ARROW-10188](https://issues.apache.org/jira/browse/ARROW-10188) - [Rust] [DataFusion] Some examples are broken +* [ARROW-10189](https://issues.apache.org/jira/browse/ARROW-10189) - [Doc] C data interface example for i32 uses \`l\`, not \`i\`, in the format +* [ARROW-10192](https://issues.apache.org/jira/browse/ARROW-10192) - [C++][Python] Segfault when converting nested struct array with dictionary field to pandas series +* [ARROW-10193](https://issues.apache.org/jira/browse/ARROW-10193) - [Python] Segfault when converting to fixed size binary array +* [ARROW-10200](https://issues.apache.org/jira/browse/ARROW-10200) - [Java][CI] Fix failure of Java CI on s390x +* [ARROW-10204](https://issues.apache.org/jira/browse/ARROW-10204) - [RUST] [Datafusion] Test failure in aggregate\_grouped\_empty with simd feature enabled +* [ARROW-10214](https://issues.apache.org/jira/browse/ARROW-10214) - [Python] UnicodeDecodeError when printing schema with binary metadata +* [ARROW-10226](https://issues.apache.org/jira/browse/ARROW-10226) - [Rust] [Parquet] Parquet reader reading wrong columns in some batches within a parquet file +* [ARROW-10230](https://issues.apache.org/jira/browse/ARROW-10230) - [JS][Doc] JavaScript documentation fails to build +* [ARROW-10232](https://issues.apache.org/jira/browse/ARROW-10232) - FixedSizeListArray is incorrectly written/read to/from parquet +* [ARROW-10234](https://issues.apache.org/jira/browse/ARROW-10234) - [C++][Gandiva] Fix logic of round() for floats/decimals in Gandiva +* [ARROW-10237](https://issues.apache.org/jira/browse/ARROW-10237) - [C++] Duplicate values in a dictionary result in corrupted parquet +* [ARROW-10238](https://issues.apache.org/jira/browse/ARROW-10238) - [C\#] List is broken +* [ARROW-10239](https://issues.apache.org/jira/browse/ARROW-10239) - [C++] aws-sdk-cpp apparently requires zlib too +* [ARROW-10244](https://issues.apache.org/jira/browse/ARROW-10244) - [Python][Docs] Add docs on using pyarrow.dataset.parquet\_dataset +* [ARROW-10248](https://issues.apache.org/jira/browse/ARROW-10248) - [C++][Dataset] Dataset writing does not write schema metadata +* [ARROW-10262](https://issues.apache.org/jira/browse/ARROW-10262) - [C++] Some TypeClass in Scalar classes seem incorrect +* [ARROW-10271](https://issues.apache.org/jira/browse/ARROW-10271) - [Rust] packed\_simd is broken and continued under a new project +* [ARROW-10279](https://issues.apache.org/jira/browse/ARROW-10279) - [Release][Python] Fix verification script to align with the new macos wheel platform tags +* [ARROW-10280](https://issues.apache.org/jira/browse/ARROW-10280) - [Packaging][Python] Fix macOS wheel artifact patterns +* [ARROW-10281](https://issues.apache.org/jira/browse/ARROW-10281) - [Python] Fix warnings when running tests +* [ARROW-10284](https://issues.apache.org/jira/browse/ARROW-10284) - [Python] Pyarrow is raising deprecation warning about filesystems on import +* [ARROW-10285](https://issues.apache.org/jira/browse/ARROW-10285) - [Python] pyarrow.orc submodule is using deprecated functionality +* [ARROW-10286](https://issues.apache.org/jira/browse/ARROW-10286) - [C++][Flight] Misleading CMake errors +* [ARROW-10288](https://issues.apache.org/jira/browse/ARROW-10288) - [C++] Compilation fails on i386 +* [ARROW-10290](https://issues.apache.org/jira/browse/ARROW-10290) - [C++] List POP\_BACK is not available in older CMake versions + + +## New Features and Improvements + +* [ARROW-983](https://issues.apache.org/jira/browse/ARROW-983) - [C++] Implement InputStream and OutputStream classes for interacting with socket connections +* [ARROW-1105](https://issues.apache.org/jira/browse/ARROW-1105) - [C++] SQLite record batch reader +* [ARROW-1509](https://issues.apache.org/jira/browse/ARROW-1509) - [Python] Write serialized object as a stream of encapsulated IPC messages +* [ARROW-1669](https://issues.apache.org/jira/browse/ARROW-1669) - [C++] Consider adding Abseil (Google C++11 standard library extensions) to toolchain +* [ARROW-1797](https://issues.apache.org/jira/browse/ARROW-1797) - [C++] Implement binary arithmetic kernels for numeric arrays +* [ARROW-2164](https://issues.apache.org/jira/browse/ARROW-2164) - [C++] Clean up unnecessary decimal module refs +* [ARROW-3080](https://issues.apache.org/jira/browse/ARROW-3080) - [Python] Unify Arrow to Python object conversion paths +* [ARROW-3757](https://issues.apache.org/jira/browse/ARROW-3757) - [R] R bindings for Flight RPC client +* [ARROW-3872](https://issues.apache.org/jira/browse/ARROW-3872) - [R] Add ad hoc test of feather compatibility +* [ARROW-4046](https://issues.apache.org/jira/browse/ARROW-4046) - [Python/CI] Exercise large memory tests +* [ARROW-4248](https://issues.apache.org/jira/browse/ARROW-4248) - [C++][Plasma] Build on Windows / Visual Studio +* [ARROW-4685](https://issues.apache.org/jira/browse/ARROW-4685) - [C++] Update Boost to 1.69 in manylinux1 docker image +* [ARROW-4927](https://issues.apache.org/jira/browse/ARROW-4927) - [Rust] Update top level README to describe current functionality +* [ARROW-4957](https://issues.apache.org/jira/browse/ARROW-4957) - [Rust] [DataFusion] Implement get\_supertype correctly +* [ARROW-4965](https://issues.apache.org/jira/browse/ARROW-4965) - [Python] Timestamp array type detection should use tzname of datetime.datetime objects +* [ARROW-5034](https://issues.apache.org/jira/browse/ARROW-5034) - [C\#] ArrowStreamWriter should expose synchronous Write methods +* [ARROW-5123](https://issues.apache.org/jira/browse/ARROW-5123) - [Rust] derive RecordWriter from struct definitions +* [ARROW-6075](https://issues.apache.org/jira/browse/ARROW-6075) - [FlightRPC] Handle uncaught exceptions in middleware +* [ARROW-6281](https://issues.apache.org/jira/browse/ARROW-6281) - [Python] Produce chunked arrays for nested types in pyarrow.array +* [ARROW-6282](https://issues.apache.org/jira/browse/ARROW-6282) - [Format] Support lossy compression +* [ARROW-6437](https://issues.apache.org/jira/browse/ARROW-6437) - [R] Add AWS SDK to system dependencies for macOS and Windows +* [ARROW-6535](https://issues.apache.org/jira/browse/ARROW-6535) - [C++] Status::WithMessage should accept variadic parameters +* [ARROW-6537](https://issues.apache.org/jira/browse/ARROW-6537) - [R] Pass column\_types to CSV reader +* [ARROW-6972](https://issues.apache.org/jira/browse/ARROW-6972) - [C\#] Should support StructField arrays +* [ARROW-6982](https://issues.apache.org/jira/browse/ARROW-6982) - [R] Add bindings for compare and boolean kernels +* [ARROW-7136](https://issues.apache.org/jira/browse/ARROW-7136) - [Rust][CI] Pre-install the rust dependencies in the dockerfile +* [ARROW-7218](https://issues.apache.org/jira/browse/ARROW-7218) - [Python] Conversion from boolean numpy scalars not working +* [ARROW-7302](https://issues.apache.org/jira/browse/ARROW-7302) - [C++] CSV: allow converting a column to a specific dictionary type +* [ARROW-7372](https://issues.apache.org/jira/browse/ARROW-7372) - [C++] Allow creating dictionary array from simple JSON +* [ARROW-7871](https://issues.apache.org/jira/browse/ARROW-7871) - [Python] Expose more compute kernels +* [ARROW-7960](https://issues.apache.org/jira/browse/ARROW-7960) - [C++][Parquet] Add support for schema translation from parquet nodes back to arrow for missing types +* [ARROW-8001](https://issues.apache.org/jira/browse/ARROW-8001) - [R][Dataset] Bindings for dataset writing +* [ARROW-8002](https://issues.apache.org/jira/browse/ARROW-8002) - [C++][Dataset] Dataset writing should let you (re)partition the data +* [ARROW-8048](https://issues.apache.org/jira/browse/ARROW-8048) - [Python] Run memory leak tests nightly as follow up to ARROW-4120 +* [ARROW-8172](https://issues.apache.org/jira/browse/ARROW-8172) - [C++] ArrayFromJSON for dictionary arrays +* [ARROW-8205](https://issues.apache.org/jira/browse/ARROW-8205) - [Rust] [DataFusion] DataFusion should enforce unique field names in a schema +* [ARROW-8253](https://issues.apache.org/jira/browse/ARROW-8253) - [Rust] [DataFusion] Improve ergonomics of registering UDFs +* [ARROW-8262](https://issues.apache.org/jira/browse/ARROW-8262) - [Rust] [DataFusion] Add example that uses LogicalPlanBuilder +* [ARROW-8289](https://issues.apache.org/jira/browse/ARROW-8289) - [Rust] [Parquet] Implement minimal Arrow Parquet writer as starting point for full writer +* [ARROW-8296](https://issues.apache.org/jira/browse/ARROW-8296) - [C++][Dataset] IpcFileFormat should support writing files with compressed buffers +* [ARROW-8355](https://issues.apache.org/jira/browse/ARROW-8355) - [Python] Reduce the number of pandas dependent test cases in test\_feather +* [ARROW-8359](https://issues.apache.org/jira/browse/ARROW-8359) - [C++/Python] Enable aarch64/ppc64le build in conda recipes +* [ARROW-8383](https://issues.apache.org/jira/browse/ARROW-8383) - [Rust] Easier random access to DictionaryArray keys and values +* [ARROW-8402](https://issues.apache.org/jira/browse/ARROW-8402) - [Java] Support ValidateFull methods in Java +* [ARROW-8423](https://issues.apache.org/jira/browse/ARROW-8423) - [Rust] [Parquet] Serialize arrow schema into metadata when writing parquet +* [ARROW-8426](https://issues.apache.org/jira/browse/ARROW-8426) - [Rust] [Parquet] Add support for writing dictionary types +* [ARROW-8493](https://issues.apache.org/jira/browse/ARROW-8493) - [C++] Create unified schema resolution code for Array reconstruction. +* [ARROW-8494](https://issues.apache.org/jira/browse/ARROW-8494) - [C++] Implement basic array-by-array reassembly logic +* [ARROW-8581](https://issues.apache.org/jira/browse/ARROW-8581) - [C\#] Date32/64Array.Builder should accept DateTime, not DateTimeOffset +* [ARROW-8601](https://issues.apache.org/jira/browse/ARROW-8601) - [Go][Flight] Implement Flight Writer interface +* [ARROW-8601](https://issues.apache.org/jira/browse/ARROW-8601) - [Go][Flight] Implement Flight Writer interface +* [ARROW-8618](https://issues.apache.org/jira/browse/ARROW-8618) - [C++] ASSIGN\_OR\_RAISE should move its argument +* [ARROW-8678](https://issues.apache.org/jira/browse/ARROW-8678) - [C++][Parquet] Remove legacy arrow to level translation. +* [ARROW-8712](https://issues.apache.org/jira/browse/ARROW-8712) - [R] Expose strptime timestamp parsing in read\_csv conversion options +* [ARROW-8774](https://issues.apache.org/jira/browse/ARROW-8774) - [Rust] [DataFusion] Improve threading model +* [ARROW-8810](https://issues.apache.org/jira/browse/ARROW-8810) - [R] Add documentation about Parquet format, appending to stream format +* [ARROW-8824](https://issues.apache.org/jira/browse/ARROW-8824) - [Rust] [DataFusion] Implement new SQL parser +* [ARROW-8828](https://issues.apache.org/jira/browse/ARROW-8828) - [Rust] Implement SQL tokenizer +* [ARROW-8829](https://issues.apache.org/jira/browse/ARROW-8829) - [Rust] Implement SQL parser +* [ARROW-9010](https://issues.apache.org/jira/browse/ARROW-9010) - [Java] Framework and interface changes for RecordBatch IPC buffer compression +* [ARROW-9065](https://issues.apache.org/jira/browse/ARROW-9065) - [C++] Support parsing date32 in dataset partition folders +* [ARROW-9068](https://issues.apache.org/jira/browse/ARROW-9068) - [C++][Dataset] Simplify Partitioning interface +* [ARROW-9078](https://issues.apache.org/jira/browse/ARROW-9078) - [C++] Parquet writing of extension type with nested storage type fails +* [ARROW-9104](https://issues.apache.org/jira/browse/ARROW-9104) - [C++] Parquet encryption tests should write files to a temporary directory instead of the testing submodule's directory +* [ARROW-9107](https://issues.apache.org/jira/browse/ARROW-9107) - [C++][Dataset] Time-based types support +* [ARROW-9147](https://issues.apache.org/jira/browse/ARROW-9147) - [C++][Dataset] Support null -\> other type promotion in Dataset scanning +* [ARROW-9205](https://issues.apache.org/jira/browse/ARROW-9205) - [Documentation] Fix typos in Columnar.rst +* [ARROW-9266](https://issues.apache.org/jira/browse/ARROW-9266) - [Python][Packaging] Enable S3 support in macOS wheels +* [ARROW-9271](https://issues.apache.org/jira/browse/ARROW-9271) - [R] Preserve data frame metadata in round trip +* [ARROW-9286](https://issues.apache.org/jira/browse/ARROW-9286) - [C++] Add function "aliases" to compute::FunctionRegistry +* [ARROW-9328](https://issues.apache.org/jira/browse/ARROW-9328) - [C++][Gandiva] Add LTRIM, RTRIM, BTRIM functions for string +* [ARROW-9338](https://issues.apache.org/jira/browse/ARROW-9338) - [Rust] Add instructions for running clippy locally +* [ARROW-9344](https://issues.apache.org/jira/browse/ARROW-9344) - [C++][Flight] measure latency quantile in flight benchmark +* [ARROW-9358](https://issues.apache.org/jira/browse/ARROW-9358) - [Integration] Reconsider generated\_large\_batch.json +* [ARROW-9371](https://issues.apache.org/jira/browse/ARROW-9371) - [Java] Run vector tests for both allocators +* [ARROW-9377](https://issues.apache.org/jira/browse/ARROW-9377) - [Java] Support unsigned dictionary indices +* [ARROW-9387](https://issues.apache.org/jira/browse/ARROW-9387) - [R] Use new C++ table select method +* [ARROW-9388](https://issues.apache.org/jira/browse/ARROW-9388) - [C++] Division kernels +* [ARROW-9394](https://issues.apache.org/jira/browse/ARROW-9394) - [Python] Support pickling of Scalars +* [ARROW-9398](https://issues.apache.org/jira/browse/ARROW-9398) - [C++] Register the SIMD sum variants under function instance instead a SIMD function +* [ARROW-9402](https://issues.apache.org/jira/browse/ARROW-9402) - [C++] Add portable wrappers for \_\_builtin\_add\_overflow and friends +* [ARROW-9405](https://issues.apache.org/jira/browse/ARROW-9405) - [R] Switch to cpp11 +* [ARROW-9412](https://issues.apache.org/jira/browse/ARROW-9412) - [C++] Add non-BUNDLED dependencies to exported INSTALL\_INTERFACE\_LIBS of arrow\_static and test that it works +* [ARROW-9429](https://issues.apache.org/jira/browse/ARROW-9429) - [Python] ChunkedArray.to\_numpy +* [ARROW-9454](https://issues.apache.org/jira/browse/ARROW-9454) - [GLib] Add binding of some dictionary builders +* [ARROW-9465](https://issues.apache.org/jira/browse/ARROW-9465) - [Python] Improve ergonomics of compute functions +* [ARROW-9469](https://issues.apache.org/jira/browse/ARROW-9469) - [Python] Make more objects weakrefable +* [ARROW-9487](https://issues.apache.org/jira/browse/ARROW-9487) - [Developer] Cover the archery release utilities with unittests +* [ARROW-9488](https://issues.apache.org/jira/browse/ARROW-9488) - [Release] Use the new changelog generation when updating the website +* [ARROW-9507](https://issues.apache.org/jira/browse/ARROW-9507) - [Rust] [DataFusion] PhysicalExpr should implement Display trait +* [ARROW-9508](https://issues.apache.org/jira/browse/ARROW-9508) - [Release][APT][Yum] Enable verification for arm64 binaries +* [ARROW-9516](https://issues.apache.org/jira/browse/ARROW-9516) - [Rust][DataFusion] Refactor physical expressions to not care about their names nor indexes +* [ARROW-9517](https://issues.apache.org/jira/browse/ARROW-9517) - [C++][Python] Allow session\_token argument when initializing S3FileSystem +* [ARROW-9518](https://issues.apache.org/jira/browse/ARROW-9518) - [Python] Deprecate pyarrow serialization +* [ARROW-9521](https://issues.apache.org/jira/browse/ARROW-9521) - [Rust] CsvReadOptions should allow file extension to be specified +* [ARROW-9523](https://issues.apache.org/jira/browse/ARROW-9523) - [Rust] improve performance of filter kernel +* [ARROW-9534](https://issues.apache.org/jira/browse/ARROW-9534) - [Rust] [DataFusion] Implement functions for creating literal expressions for all types +* [ARROW-9550](https://issues.apache.org/jira/browse/ARROW-9550) - [Rust] [DataFusion] Remove Rc\> from hash aggregate operator +* [ARROW-9553](https://issues.apache.org/jira/browse/ARROW-9553) - [Rust] Release script doesn't bump parquet crate's arrow dependency version +* [ARROW-9557](https://issues.apache.org/jira/browse/ARROW-9557) - [R] Iterating over parquet columns is slow in R +* [ARROW-9559](https://issues.apache.org/jira/browse/ARROW-9559) - [Rust] [DataFusion] Revert privatization of exprlist\_to\_fields +* [ARROW-9563](https://issues.apache.org/jira/browse/ARROW-9563) - [Dev][Release] Use archery's changelog generator when creating release notes for the website +* [ARROW-9568](https://issues.apache.org/jira/browse/ARROW-9568) - [CI] Use official msys action on GHA +* [ARROW-9576](https://issues.apache.org/jira/browse/ARROW-9576) - [Python][Doc] Fix error in code example for extension types +* [ARROW-9580](https://issues.apache.org/jira/browse/ARROW-9580) - [JS] Docs have superfluous () +* [ARROW-9581](https://issues.apache.org/jira/browse/ARROW-9581) - [Dev][Release] Bump next snapshot versions to 2.0.0 +* [ARROW-9582](https://issues.apache.org/jira/browse/ARROW-9582) - [Rust] Implement Array::memory\_size() +* [ARROW-9585](https://issues.apache.org/jira/browse/ARROW-9585) - [Rust] Remove duplicated to-do line in DataFusion readme +* [ARROW-9587](https://issues.apache.org/jira/browse/ARROW-9587) - [FlightRPC][Java] Clean up DoPut/FlightStream memory handling +* [ARROW-9593](https://issues.apache.org/jira/browse/ARROW-9593) - [Python] Add custom pickle reducers for DictionaryScalar +* [ARROW-9604](https://issues.apache.org/jira/browse/ARROW-9604) - [C++] Add benchmark for aggregate min/max compute kernels +* [ARROW-9605](https://issues.apache.org/jira/browse/ARROW-9605) - [C++] Optimize performance for aggregate min/max compute kernels +* [ARROW-9607](https://issues.apache.org/jira/browse/ARROW-9607) - [C++][Gandiva] Add bitwise\_and(), bitwise\_or() and bitwise\_not() functions for integers +* [ARROW-9608](https://issues.apache.org/jira/browse/ARROW-9608) - [Rust] Remove arrow flight from parquet's feature gating +* [ARROW-9615](https://issues.apache.org/jira/browse/ARROW-9615) - [Rust] Add kernel to compute length of string array +* [ARROW-9617](https://issues.apache.org/jira/browse/ARROW-9617) - [Rust] [DataFusion] Add length of string array +* [ARROW-9618](https://issues.apache.org/jira/browse/ARROW-9618) - [Rust] [DataFusion] Make it easier to write optimizers +* [ARROW-9619](https://issues.apache.org/jira/browse/ARROW-9619) - [Rust] [DataFusion] Add predicate push-down +* [ARROW-9632](https://issues.apache.org/jira/browse/ARROW-9632) - [Rust] Add a "new" method for ExecutionContextSchemaProvider +* [ARROW-9638](https://issues.apache.org/jira/browse/ARROW-9638) - [C++][Compute] Implement mode(most frequent number) kernel +* [ARROW-9639](https://issues.apache.org/jira/browse/ARROW-9639) - [Ruby] Add dependency version check +* [ARROW-9640](https://issues.apache.org/jira/browse/ARROW-9640) - [C++][Gandiva] Implement round() for integers and long integers +* [ARROW-9641](https://issues.apache.org/jira/browse/ARROW-9641) - [C++][Gandiva] Implement round() for floating point and double floating point numbers +* [ARROW-9645](https://issues.apache.org/jira/browse/ARROW-9645) - [Python] Deprecate the legacy pyarrow.filesystem interface +* [ARROW-9646](https://issues.apache.org/jira/browse/ARROW-9646) - [C++][Dataset] Add support for writing parquet datasets +* [ARROW-9650](https://issues.apache.org/jira/browse/ARROW-9650) - [Packaging][APT] Drop support for Ubuntu 19.10 +* [ARROW-9654](https://issues.apache.org/jira/browse/ARROW-9654) - [Rust][DataFusion] Add an EXPLAIN command to the datafusion CLI +* [ARROW-9656](https://issues.apache.org/jira/browse/ARROW-9656) - [Rust][DataFusion] Slightly confusing error message when unsupported type is provided to CREATE EXTERNAL TABLE +* [ARROW-9658](https://issues.apache.org/jira/browse/ARROW-9658) - [Python][Dataset] Bindings for dataset writing +* [ARROW-9665](https://issues.apache.org/jira/browse/ARROW-9665) - [R] head/tail/take for Datasets +* [ARROW-9667](https://issues.apache.org/jira/browse/ARROW-9667) - [CI][Crossbow] Segfault in 2 nightly R builds +* [ARROW-9671](https://issues.apache.org/jira/browse/ARROW-9671) - [C++] BasicDecimal128 constructor interprets uint64\_t integers with highest bit set as negative +* [ARROW-9673](https://issues.apache.org/jira/browse/ARROW-9673) - [Rust] Add a param "dialect" for DFParser::parse\_sql +* [ARROW-9678](https://issues.apache.org/jira/browse/ARROW-9678) - [Rust] [DataFusion] Improve projection push down to remove unused columns +* [ARROW-9679](https://issues.apache.org/jira/browse/ARROW-9679) - [Rust] [DataFusion] HashAggregate walks map many times building final batch +* [ARROW-9681](https://issues.apache.org/jira/browse/ARROW-9681) - [Java] Failed Arrow Memory - Core on big-endian platform +* [ARROW-9683](https://issues.apache.org/jira/browse/ARROW-9683) - [Rust][DataFusion] Implement Debug for ExecutionPlan trait +* [ARROW-9691](https://issues.apache.org/jira/browse/ARROW-9691) - [Rust] [DataFusion] Make sql\_statement\_to\_plan public +* [ARROW-9695](https://issues.apache.org/jira/browse/ARROW-9695) - [Rust][DataFusion] Improve documentation on LogicalPlan variants +* [ARROW-9699](https://issues.apache.org/jira/browse/ARROW-9699) - [C++][Compute] Improve mode kernel performance for small integer types +* [ARROW-9701](https://issues.apache.org/jira/browse/ARROW-9701) - [Java][CI] Add a test job on s390x +* [ARROW-9702](https://issues.apache.org/jira/browse/ARROW-9702) - [C++] Move bpacking simd to runtime path +* [ARROW-9703](https://issues.apache.org/jira/browse/ARROW-9703) - [Developer][Archery] Restartable cherry-picking process for creating maintenance branches +* [ARROW-9706](https://issues.apache.org/jira/browse/ARROW-9706) - [Java] Tests in TestLargeListVector fails on big endian platform +* [ARROW-9710](https://issues.apache.org/jira/browse/ARROW-9710) - [C++] Generalize Decimal ToString in preparation for Decimal256 +* [ARROW-9711](https://issues.apache.org/jira/browse/ARROW-9711) - [Rust] Add benchmark based on TPC-H +* [ARROW-9713](https://issues.apache.org/jira/browse/ARROW-9713) - [Rust][DataFusion] Remove explicit panics +* [ARROW-9715](https://issues.apache.org/jira/browse/ARROW-9715) - [R] changelog/doc updates for 1.0.1 +* [ARROW-9718](https://issues.apache.org/jira/browse/ARROW-9718) - [Python] Make pyarrow.parquet work with the new filesystem interfaces +* [ARROW-9721](https://issues.apache.org/jira/browse/ARROW-9721) - [Packaging][Python] Update wheel dependency files +* [ARROW-9722](https://issues.apache.org/jira/browse/ARROW-9722) - [Rust]: Shorten key lifetime for reverse lookup for dictionary arrays +* [ARROW-9723](https://issues.apache.org/jira/browse/ARROW-9723) - [C++] Expected behaviour of "mode" kernel with NaNs ? +* [ARROW-9725](https://issues.apache.org/jira/browse/ARROW-9725) - [Rust] [DataFusion] LimitExec and SortExec should use MergeExec +* [ARROW-9737](https://issues.apache.org/jira/browse/ARROW-9737) - [C++][Gandiva] Add bitwise\_xor() for integers +* [ARROW-9739](https://issues.apache.org/jira/browse/ARROW-9739) - [CI][Ruby] Don't install gem documents +* [ARROW-9742](https://issues.apache.org/jira/browse/ARROW-9742) - [Rust] Create one standard DataFrame API +* [ARROW-9751](https://issues.apache.org/jira/browse/ARROW-9751) - [Rust] [DataFusion] Extend UDFs to accept more than one type per argument +* [ARROW-9752](https://issues.apache.org/jira/browse/ARROW-9752) - [Rust] [DataFusion] Add support for Aggregate UDFs +* [ARROW-9753](https://issues.apache.org/jira/browse/ARROW-9753) - [Rust] [DataFusion] Remove the use of Mutex in ExecutionPlan trait +* [ARROW-9754](https://issues.apache.org/jira/browse/ARROW-9754) - [Rust] [DataFusion] Implement async in DataFusion traits +* [ARROW-9757](https://issues.apache.org/jira/browse/ARROW-9757) - [Rust] [DataFusion] Use "pub use" to expose a clean public API +* [ARROW-9758](https://issues.apache.org/jira/browse/ARROW-9758) - [Rust] [DataFusion] Implement extension API for DataFusion +* [ARROW-9759](https://issues.apache.org/jira/browse/ARROW-9759) - [Rust] [DataFusion] Implement DataFrame::sort +* [ARROW-9760](https://issues.apache.org/jira/browse/ARROW-9760) - [Rust] [DataFusion] Implement DataFrame::explain +* [ARROW-9761](https://issues.apache.org/jira/browse/ARROW-9761) - [C++] Add experimental pull-based iterator structures to C interface implementation +* [ARROW-9762](https://issues.apache.org/jira/browse/ARROW-9762) - [Rust] [DataFusion] ExecutionContext::sql should return DataFrame +* [ARROW-9769](https://issues.apache.org/jira/browse/ARROW-9769) - [Python] Remove skip for in-memory fsspec in test\_move\_file +* [ARROW-9775](https://issues.apache.org/jira/browse/ARROW-9775) - [C++] Automatic S3 region selection +* [ARROW-9781](https://issues.apache.org/jira/browse/ARROW-9781) - [C++] Fix uninitialized value warnings +* [ARROW-9782](https://issues.apache.org/jira/browse/ARROW-9782) - [C++][Dataset] Ability to write ".feather" files with IpcFileFormat +* [ARROW-9784](https://issues.apache.org/jira/browse/ARROW-9784) - [Rust] [DataFusion] Improve instructions for running tpch benchmark +* [ARROW-9786](https://issues.apache.org/jira/browse/ARROW-9786) - [R] Unvendor cpp11 before release +* [ARROW-9788](https://issues.apache.org/jira/browse/ARROW-9788) - Handle naming inconsistencies between SQL, DataFrame API and struct names +* [ARROW-9792](https://issues.apache.org/jira/browse/ARROW-9792) - [Rust] [DataFusion] Logical aggregate functions should not return Result +* [ARROW-9794](https://issues.apache.org/jira/browse/ARROW-9794) - [C++] Add functionality to cpu\_info to discriminate between Intel vs AMD x86 +* [ARROW-9795](https://issues.apache.org/jira/browse/ARROW-9795) - [C++][Gandiva] Implement castTIMESTAMP(int64) in Gandiva +* [ARROW-9806](https://issues.apache.org/jira/browse/ARROW-9806) - [R] More compute kernel bindings +* [ARROW-9807](https://issues.apache.org/jira/browse/ARROW-9807) - [R] News update/version bump post-1.0.1 +* [ARROW-9808](https://issues.apache.org/jira/browse/ARROW-9808) - [Python] parquet.read\_table docstring wrong use\_legacy\_dataset explanation +* [ARROW-9811](https://issues.apache.org/jira/browse/ARROW-9811) - [C++] Unchecked floating point division by 0 should succeed +* [ARROW-9813](https://issues.apache.org/jira/browse/ARROW-9813) - [C++] Disable semantic interposition +* [ARROW-9819](https://issues.apache.org/jira/browse/ARROW-9819) - [C++] Bump mimalloc to 1.6.4 +* [ARROW-9821](https://issues.apache.org/jira/browse/ARROW-9821) - [Rust][DataFusion] User Defined PlanNode / Operator API +* [ARROW-9821](https://issues.apache.org/jira/browse/ARROW-9821) - [Rust][DataFusion] User Defined PlanNode / Operator API +* [ARROW-9823](https://issues.apache.org/jira/browse/ARROW-9823) - [CI][C++][MinGW] Enable S3 +* [ARROW-9832](https://issues.apache.org/jira/browse/ARROW-9832) - [Rust] [DataFusion] Refactor PhysicalPlan to remove Partition +* [ARROW-9833](https://issues.apache.org/jira/browse/ARROW-9833) - [Rust] [DataFusion] Refactor TableProvider.scan to return ExecutionPlan +* [ARROW-9834](https://issues.apache.org/jira/browse/ARROW-9834) - [Rust] [DataFusion] Remove Partition trait +* [ARROW-9835](https://issues.apache.org/jira/browse/ARROW-9835) - [Rust] [DataFusion] Remove FunctionMeta +* [ARROW-9836](https://issues.apache.org/jira/browse/ARROW-9836) - [Rust] [DataFusion] Improve API for usage of UDFs +* [ARROW-9837](https://issues.apache.org/jira/browse/ARROW-9837) - [Rust] Add provider for variable +* [ARROW-9838](https://issues.apache.org/jira/browse/ARROW-9838) - [Rust] [DataFusion] DefaultPhysicalPlanner should insert explicit MergeExec nodes +* [ARROW-9839](https://issues.apache.org/jira/browse/ARROW-9839) - [Rust] [DataFusion] Add ability to downcast ExecutionPlan to specific operator +* [ARROW-9841](https://issues.apache.org/jira/browse/ARROW-9841) - [Rust] Update checked-in flatbuffer files +* [ARROW-9844](https://issues.apache.org/jira/browse/ARROW-9844) - [Go][CI] Add Travis CI job for Go on s390x +* [ARROW-9845](https://issues.apache.org/jira/browse/ARROW-9845) - [Rust] [Parquet] serde\_json is only used in tests but isn't in dev-dependencies +* [ARROW-9848](https://issues.apache.org/jira/browse/ARROW-9848) - [Rust] Implement changes to ensure flatbuffer alignment +* [ARROW-9849](https://issues.apache.org/jira/browse/ARROW-9849) - [Rust] [DataFusion] Make UDFs not need a Field +* [ARROW-9850](https://issues.apache.org/jira/browse/ARROW-9850) - [Go] Defer should not be used in the loop +* [ARROW-9853](https://issues.apache.org/jira/browse/ARROW-9853) - [RUST] Implement "take" kernel for dictionary arrays +* [ARROW-9854](https://issues.apache.org/jira/browse/ARROW-9854) - [R] Support reading/writing data to/from S3 +* [ARROW-9858](https://issues.apache.org/jira/browse/ARROW-9858) - [C++][Python][Docs] Expand user guide for FileSystem +* [ARROW-9863](https://issues.apache.org/jira/browse/ARROW-9863) - [C++] [PARQUET] Optimize meta data recovery of ApplicationVersion +* [ARROW-9867](https://issues.apache.org/jira/browse/ARROW-9867) - [C++][Dataset] FileSystemDataset should expose its filesystem +* [ARROW-9868](https://issues.apache.org/jira/browse/ARROW-9868) - [C++] Provide utility for copying files between filesystems +* [ARROW-9869](https://issues.apache.org/jira/browse/ARROW-9869) - [R] Implement full S3FileSystem/S3Options constructor +* [ARROW-9870](https://issues.apache.org/jira/browse/ARROW-9870) - [R] Friendly interface for filesystems (S3) +* [ARROW-9871](https://issues.apache.org/jira/browse/ARROW-9871) - [C++] Add uppercase support to ARROW\_USER\_SIMD\_LEVEL. +* [ARROW-9873](https://issues.apache.org/jira/browse/ARROW-9873) - [C++][Compute] Improve mode kernel for intergers within limited value range +* [ARROW-9875](https://issues.apache.org/jira/browse/ARROW-9875) - [Python] Let FileSystem.get\_file\_info accept a single path +* [ARROW-9884](https://issues.apache.org/jira/browse/ARROW-9884) - [R] Bindings for writing datasets to Parquet +* [ARROW-9885](https://issues.apache.org/jira/browse/ARROW-9885) - [Rust] [DataFusion] Simplify code of type coercion for binary types +* [ARROW-9886](https://issues.apache.org/jira/browse/ARROW-9886) - [Rust] [DataFusion] Simplify code to test cast +* [ARROW-9887](https://issues.apache.org/jira/browse/ARROW-9887) - [Rust] [DataFusion] Add support for complex return types of built-in functions +* [ARROW-9890](https://issues.apache.org/jira/browse/ARROW-9890) - [R] Add zstandard compression codec in macOS build +* [ARROW-9891](https://issues.apache.org/jira/browse/ARROW-9891) - [Rust] [DataFusion] Make math functions support f32 +* [ARROW-9892](https://issues.apache.org/jira/browse/ARROW-9892) - [Rust] [DataFusion] Add support for concat +* [ARROW-9893](https://issues.apache.org/jira/browse/ARROW-9893) - [Python] Bindings for writing datasets to Parquet +* [ARROW-9895](https://issues.apache.org/jira/browse/ARROW-9895) - [RUST] Improve sort kernels +* [ARROW-9899](https://issues.apache.org/jira/browse/ARROW-9899) - [Rust] [DataFusion] Switch from Box --\> SchemaRef (Arc) to be consistent with the rest of Arrow +* [ARROW-9900](https://issues.apache.org/jira/browse/ARROW-9900) - [Rust][DataFusion] Use Arc<\> instead of Box<\> in LogicalPlan +* [ARROW-9901](https://issues.apache.org/jira/browse/ARROW-9901) - [C++] Add hand-crafted Parquet to Arrow reconstruction test for nested reading +* [ARROW-9902](https://issues.apache.org/jira/browse/ARROW-9902) - [Rust] [DataFusion] Add support for array() +* [ARROW-9904](https://issues.apache.org/jira/browse/ARROW-9904) - [C++] Unroll the loop manually for CountSetBits +* [ARROW-9908](https://issues.apache.org/jira/browse/ARROW-9908) - [Rust] Support temporal data types in JSON reader +* [ARROW-9910](https://issues.apache.org/jira/browse/ARROW-9910) - [Rust] [DataFusion] Type coercion of Variadic is wrong +* [ARROW-9914](https://issues.apache.org/jira/browse/ARROW-9914) - [Rust][DataFusion] Document the SQL -\> Arrow type mapping +* [ARROW-9916](https://issues.apache.org/jira/browse/ARROW-9916) - [RUST] Avoid cloning ArrayData in several places +* [ARROW-9917](https://issues.apache.org/jira/browse/ARROW-9917) - [Python][Compute] Add bindings for mode kernel +* [ARROW-9919](https://issues.apache.org/jira/browse/ARROW-9919) - [Rust] [DataFusion] Math functions +* [ARROW-9921](https://issues.apache.org/jira/browse/ARROW-9921) - [Rust] Add \`from(Vec\>)\` to [Large]StringArray +* [ARROW-9925](https://issues.apache.org/jira/browse/ARROW-9925) - [GLib] Add low level value readers for GArrowListArray family +* [ARROW-9926](https://issues.apache.org/jira/browse/ARROW-9926) - [GLib] Use placement new for GArrowRecordBatchFileReader +* [ARROW-9928](https://issues.apache.org/jira/browse/ARROW-9928) - [C++] Speed up integer parsing slightly +* [ARROW-9929](https://issues.apache.org/jira/browse/ARROW-9929) - [Developer] Autotune cmake-format +* [ARROW-9933](https://issues.apache.org/jira/browse/ARROW-9933) - [Developer] Add drone as a CI provider for crossbow +* [ARROW-9934](https://issues.apache.org/jira/browse/ARROW-9934) - [Rust] Shape and stride check in tensor +* [ARROW-9941](https://issues.apache.org/jira/browse/ARROW-9941) - [Python] Better string representation for extension types +* [ARROW-9944](https://issues.apache.org/jira/browse/ARROW-9944) - [Rust] Implement TO\_TIMESTAMP function +* [ARROW-9949](https://issues.apache.org/jira/browse/ARROW-9949) - [C++] Generalize Decimal128::FromString for reuse in Decimal256 +* [ARROW-9950](https://issues.apache.org/jira/browse/ARROW-9950) - [Rust] [DataFusion] Allow UDF usage without registry +* [ARROW-9952](https://issues.apache.org/jira/browse/ARROW-9952) - [Python] Use pyarrow.dataset writing for pq.write\_to\_dataset +* [ARROW-9954](https://issues.apache.org/jira/browse/ARROW-9954) - [Rust] [DataFusion] Simplify code of aggregate planning +* [ARROW-9956](https://issues.apache.org/jira/browse/ARROW-9956) - [C++][Gandiva] Implement Binary string function in Gandiva +* [ARROW-9957](https://issues.apache.org/jira/browse/ARROW-9957) - [Rust] Remove unmaintained tempdir dependency +* [ARROW-9961](https://issues.apache.org/jira/browse/ARROW-9961) - [Rust][DataFusion] to\_timestamp function parses timestamp without timezone offset as UTC rather than local +* [ARROW-9964](https://issues.apache.org/jira/browse/ARROW-9964) - [C++] CSV date support +* [ARROW-9965](https://issues.apache.org/jira/browse/ARROW-9965) - [Java] Buffer capacity calculations are slow for fixed-width vectors +* [ARROW-9966](https://issues.apache.org/jira/browse/ARROW-9966) - [Rust] Speedup aggregate kernels +* [ARROW-9967](https://issues.apache.org/jira/browse/ARROW-9967) - [Python] Add compute module docs +* [ARROW-9971](https://issues.apache.org/jira/browse/ARROW-9971) - [Rust] Speedup take +* [ARROW-9977](https://issues.apache.org/jira/browse/ARROW-9977) - [Rust] Add min/max for [Large]String +* [ARROW-9979](https://issues.apache.org/jira/browse/ARROW-9979) - [Rust] Fix arrow crate clippy lints +* [ARROW-9980](https://issues.apache.org/jira/browse/ARROW-9980) - [Rust] Fix parquet crate clippy lints +* [ARROW-9981](https://issues.apache.org/jira/browse/ARROW-9981) - [Rust] Allow configuring flight IPC with IpcWriteOptions +* [ARROW-9983](https://issues.apache.org/jira/browse/ARROW-9983) - [C++][Dataset][Python] Use larger default batch size than 32K for Datasets API +* [ARROW-9984](https://issues.apache.org/jira/browse/ARROW-9984) - [Rust] [DataFusion] DRY of function to string +* [ARROW-9986](https://issues.apache.org/jira/browse/ARROW-9986) - [Rust][DataFusion] TO\_TIMESTAMP function erroneously requires fractional seconds when no timezone is present +* [ARROW-9987](https://issues.apache.org/jira/browse/ARROW-9987) - [Rust] [DataFusion] Improve docs of \`Expr\`. +* [ARROW-9988](https://issues.apache.org/jira/browse/ARROW-9988) - [Rust] [DataFusion] Added std::ops to logical expressions +* [ARROW-9992](https://issues.apache.org/jira/browse/ARROW-9992) - [C++][Python] Refactor python to arrow conversions based on a reusable conversion API +* [ARROW-9998](https://issues.apache.org/jira/browse/ARROW-9998) - [Python] Support pickling DictionaryScalar +* [ARROW-9999](https://issues.apache.org/jira/browse/ARROW-9999) - [Python] Support constructing dictionary array directly through pa.array() +* [ARROW-10000](https://issues.apache.org/jira/browse/ARROW-10000) - [C++][Python] Support constructing StructArray from list of key-value pairs +* [ARROW-10001](https://issues.apache.org/jira/browse/ARROW-10001) - [Rust] [DataFusion] Add developer guide to README +* [ARROW-10010](https://issues.apache.org/jira/browse/ARROW-10010) - [Rust] Speedup arithmetic +* [ARROW-10015](https://issues.apache.org/jira/browse/ARROW-10015) - [Rust] Implement SIMD for aggregate kernel sum +* [ARROW-10016](https://issues.apache.org/jira/browse/ARROW-10016) - [Rust] [DataFusion] Implement IsNull and IsNotNull +* [ARROW-10018](https://issues.apache.org/jira/browse/ARROW-10018) - [CI] Disable Sphinx and API documentation build since it takes 6 hours on master +* [ARROW-10019](https://issues.apache.org/jira/browse/ARROW-10019) - [Rust] Add substring kernel +* [ARROW-10023](https://issues.apache.org/jira/browse/ARROW-10023) - [Gandiva][C++] Implementing Split part function in gandiva +* [ARROW-10024](https://issues.apache.org/jira/browse/ARROW-10024) - [C++][Parquet] Create nested reading benchmarks +* [ARROW-10028](https://issues.apache.org/jira/browse/ARROW-10028) - [Rust] Simplify macro def\_numeric\_from\_vec +* [ARROW-10030](https://issues.apache.org/jira/browse/ARROW-10030) - [Rust] Support fromIter and toIter +* [ARROW-10035](https://issues.apache.org/jira/browse/ARROW-10035) - [C++] Bump versions of vendored code +* [ARROW-10037](https://issues.apache.org/jira/browse/ARROW-10037) - [C++] Workaround to force find AWS SDK to look for shared libraries +* [ARROW-10040](https://issues.apache.org/jira/browse/ARROW-10040) - [Rust] Create a way to slice unalligned offset buffers +* [ARROW-10043](https://issues.apache.org/jira/browse/ARROW-10043) - [Rust] [DataFusion] Introduce support for DISTINCT by partially implementing COUNT(DISTINCT) +* [ARROW-10044](https://issues.apache.org/jira/browse/ARROW-10044) - [Rust] Improve README +* [ARROW-10046](https://issues.apache.org/jira/browse/ARROW-10046) - [Rust] [DataFusion] Made \`\*Iterator\` implement Iterator +* [ARROW-10050](https://issues.apache.org/jira/browse/ARROW-10050) - [C++][Gandiva] Implement concat() in Gandiva for up to 10 arguments +* [ARROW-10051](https://issues.apache.org/jira/browse/ARROW-10051) - [C++][Compute] Make aggregate kernel merge state mutable +* [ARROW-10054](https://issues.apache.org/jira/browse/ARROW-10054) - [Python] Slice methods should return empty arrays instead of crashing +* [ARROW-10055](https://issues.apache.org/jira/browse/ARROW-10055) - [Rust] Implement DoubleEndedIterator for NullableIter +* [ARROW-10057](https://issues.apache.org/jira/browse/ARROW-10057) - [C++] Add Parquet-Arrow roundtrip tests for nested data +* [ARROW-10058](https://issues.apache.org/jira/browse/ARROW-10058) - [C++] Investigate performance of LevelsToBitmap without BMI2 +* [ARROW-10059](https://issues.apache.org/jira/browse/ARROW-10059) - [R][Doc] Give more advice on how to set up C++ build +* [ARROW-10063](https://issues.apache.org/jira/browse/ARROW-10063) - [Archery][CI] Fetch main branch in archery build only when it is a pull request +* [ARROW-10064](https://issues.apache.org/jira/browse/ARROW-10064) - [C++] Resolve compile warnings on Apple Clang 12 +* [ARROW-10065](https://issues.apache.org/jira/browse/ARROW-10065) - [Rust] DRY downcasted Arrays +* [ARROW-10066](https://issues.apache.org/jira/browse/ARROW-10066) - [C++] Make sure that default AWS region is respected +* [ARROW-10068](https://issues.apache.org/jira/browse/ARROW-10068) - [C++] Add bundled external project for aws-sdk-cpp +* [ARROW-10069](https://issues.apache.org/jira/browse/ARROW-10069) - [Java] Support running Java benchmarks from command line +* [ARROW-10070](https://issues.apache.org/jira/browse/ARROW-10070) - [C++][Compute] Implement stdev aggregate kernel +* [ARROW-10071](https://issues.apache.org/jira/browse/ARROW-10071) - [R] segfault with ArrowObject from previous session, or saved +* [ARROW-10074](https://issues.apache.org/jira/browse/ARROW-10074) - [C++] Don't use string\_view.to\_string() +* [ARROW-10075](https://issues.apache.org/jira/browse/ARROW-10075) - [C++] Don't use nonstd::nullopt this breaks out vendoring abstraction. +* [ARROW-10076](https://issues.apache.org/jira/browse/ARROW-10076) - [C++] Use TemporaryDir for all tests that don't already use it. +* [ARROW-10077](https://issues.apache.org/jira/browse/ARROW-10077) - [C++] Potential overflow in bit\_stream\_utils.h multiplication. +* [ARROW-10083](https://issues.apache.org/jira/browse/ARROW-10083) - [C++] Improve Parquet fuzz seed corpus +* [ARROW-10084](https://issues.apache.org/jira/browse/ARROW-10084) - [Rust] [DataFusion] Add length of large string array +* [ARROW-10086](https://issues.apache.org/jira/browse/ARROW-10086) - [Rust] Migrate min\_large\_string -\> min\_string kernels +* [ARROW-10090](https://issues.apache.org/jira/browse/ARROW-10090) - [C++][Compute] Improve mode kernel +* [ARROW-10092](https://issues.apache.org/jira/browse/ARROW-10092) - [Dev][Go] Add grpc generated go files to rat exclusion list +* [ARROW-10093](https://issues.apache.org/jira/browse/ARROW-10093) - [R] Add ability to opt-out of int64 -\> int demotion +* [ARROW-10095](https://issues.apache.org/jira/browse/ARROW-10095) - [Rust] [Parquet] Update for IPC changes +* [ARROW-10096](https://issues.apache.org/jira/browse/ARROW-10096) - [Rust] [DataFusion] Remove unused code +* [ARROW-10099](https://issues.apache.org/jira/browse/ARROW-10099) - [C++][Dataset] Also allow integer partition fields to be dictionary encoded +* [ARROW-10100](https://issues.apache.org/jira/browse/ARROW-10100) - [C++][Dataset] Ability to read/subset a ParquetFileFragment with given set of row group ids +* [ARROW-10102](https://issues.apache.org/jira/browse/ARROW-10102) - [C++] Generalize BasicDecimal128::operator\*= for reuse in Decimal256 +* [ARROW-10103](https://issues.apache.org/jira/browse/ARROW-10103) - [Rust] Add a Contains kernel +* [ARROW-10105](https://issues.apache.org/jira/browse/ARROW-10105) - [FlightRPC] Add client option to disable certificate validation with TLS +* [ARROW-10120](https://issues.apache.org/jira/browse/ARROW-10120) - [C++][Parquet] Create reading benchmarks for 2-level nested data +* [ARROW-10127](https://issues.apache.org/jira/browse/ARROW-10127) - [Format] Update specification to support 256-bit Decimal types +* [ARROW-10129](https://issues.apache.org/jira/browse/ARROW-10129) - [Rust] Cargo build is rebuilding dependencies on arrow changes +* [ARROW-10134](https://issues.apache.org/jira/browse/ARROW-10134) - [C++][Dataset] Add ParquetFileFragment::num\_row\_groups property +* [ARROW-10139](https://issues.apache.org/jira/browse/ARROW-10139) - [C++] Add support for building arrow\_testing without building tests +* [ARROW-10148](https://issues.apache.org/jira/browse/ARROW-10148) - [Rust] Add documentation to lib.rs +* [ARROW-10151](https://issues.apache.org/jira/browse/ARROW-10151) - [Python] Add support MapArray to\_pandas conversion +* [ARROW-10155](https://issues.apache.org/jira/browse/ARROW-10155) - [Rust] [DataFusion] Add documentation to lib.rs +* [ARROW-10156](https://issues.apache.org/jira/browse/ARROW-10156) - [Rust] Auto-label PRs +* [ARROW-10157](https://issues.apache.org/jira/browse/ARROW-10157) - [Rust] Add more documentation about take +* [ARROW-10160](https://issues.apache.org/jira/browse/ARROW-10160) - [Rust] Improve documentation of DictionaryType +* [ARROW-10161](https://issues.apache.org/jira/browse/ARROW-10161) - [Rust] [DataFusion] Simplify expression tests +* [ARROW-10162](https://issues.apache.org/jira/browse/ARROW-10162) - [Rust] Support display of DictionaryArrays in pretty printing +* [ARROW-10164](https://issues.apache.org/jira/browse/ARROW-10164) - [Rust] Add support for DictionaryArray types to cast kernels +* [ARROW-10167](https://issues.apache.org/jira/browse/ARROW-10167) - [Rust] Support display of DictionaryArrays in sql.rs +* [ARROW-10168](https://issues.apache.org/jira/browse/ARROW-10168) - [Rust] [Parquet] Extend arrow schema conversion to projected fields +* [ARROW-10171](https://issues.apache.org/jira/browse/ARROW-10171) - [Rust] [DataFusion] Add \`ExecutionContext::from\` +* [ARROW-10190](https://issues.apache.org/jira/browse/ARROW-10190) - [Website] Add Jorge to list of committers +* [ARROW-10191](https://issues.apache.org/jira/browse/ARROW-10191) - [Rust] [Parquet] Add roundtrip tests for single column batches +* [ARROW-10196](https://issues.apache.org/jira/browse/ARROW-10196) - [C++] Add Future::DeferNotOk() +* [ARROW-10199](https://issues.apache.org/jira/browse/ARROW-10199) - [Rust][Parquet] Release Parquet at crates.io to remove debug prints +* [ARROW-10201](https://issues.apache.org/jira/browse/ARROW-10201) - [C++][CI] Disable S3 in arm64 job on Travis CI +* [ARROW-10202](https://issues.apache.org/jira/browse/ARROW-10202) - [CI][Windows] Use sf.net mirror for MSYS2 +* [ARROW-10205](https://issues.apache.org/jira/browse/ARROW-10205) - [Java][FlightRPC] Add client option to disable server verification +* [ARROW-10206](https://issues.apache.org/jira/browse/ARROW-10206) - [Python][C++][FlightRPC] Add client option to disable server validation +* [ARROW-10215](https://issues.apache.org/jira/browse/ARROW-10215) - [Rust] [DataFusion] Rename "Source" typedef +* [ARROW-10217](https://issues.apache.org/jira/browse/ARROW-10217) - [CI] Run fewer GitHub Actions jobs +* [ARROW-10225](https://issues.apache.org/jira/browse/ARROW-10225) - [Rust] [Parquet] Fix null bitmap comparisons in roundtrip tests +* [ARROW-10227](https://issues.apache.org/jira/browse/ARROW-10227) - [Ruby] Use a table size as the default for parquet chunk\_size +* [ARROW-10229](https://issues.apache.org/jira/browse/ARROW-10229) - [C++][Parquet] Remove left over ARROW\_LOG statement. +* [ARROW-10231](https://issues.apache.org/jira/browse/ARROW-10231) - [CI] Unable to download minio in arm32v7 docker image +* [ARROW-10233](https://issues.apache.org/jira/browse/ARROW-10233) - [Rust] Make array\_value\_to\_string available in all Arrow builds +* [ARROW-10235](https://issues.apache.org/jira/browse/ARROW-10235) - [Rust][DataFusion] Improve documentation for type coercion +* [ARROW-10240](https://issues.apache.org/jira/browse/ARROW-10240) - [Rust] [Datafusion] Optionally load tpch data into memory before running benchmark query +* [ARROW-10251](https://issues.apache.org/jira/browse/ARROW-10251) - [Rust] [DataFusion] MemTable::load() should load partitions in parallel +* [ARROW-10252](https://issues.apache.org/jira/browse/ARROW-10252) - [Python] Add option to skip inclusion of Arrow headers in Python installation +* [ARROW-10256](https://issues.apache.org/jira/browse/ARROW-10256) - [C++][Flight] Disable -Werror carefully +* [ARROW-10257](https://issues.apache.org/jira/browse/ARROW-10257) - [R] Prepare news/docs for 2.0 release +* [ARROW-10260](https://issues.apache.org/jira/browse/ARROW-10260) - [Python] Missing MapType to Pandas dtype +* [ARROW-10265](https://issues.apache.org/jira/browse/ARROW-10265) - [CI] Use smaler build when cache doesn't exit on Travis CI +* [ARROW-10266](https://issues.apache.org/jira/browse/ARROW-10266) - [CI][macOS] Ensure using Python 3.8 with Homebrew +* [ARROW-10267](https://issues.apache.org/jira/browse/ARROW-10267) - [Python] Skip flight test if disable\_server\_verification feature is not available +* [ARROW-10272](https://issues.apache.org/jira/browse/ARROW-10272) - [Packaging][Python] Pin newer multibuild version to avoid updating homebrew +* [ARROW-10273](https://issues.apache.org/jira/browse/ARROW-10273) - [CI][Homebrew] Fix "brew audit" usage +* [ARROW-10287](https://issues.apache.org/jira/browse/ARROW-10287) - [C++] Avoid std::random\_device whenever possible +* [PARQUET-1845](https://issues.apache.org/jira/browse/PARQUET-1845) - [C++] Int96 memory images in test cases assume only little-endian +* [PARQUET-1878](https://issues.apache.org/jira/browse/PARQUET-1878) - [C++] lz4 codec is not compatible with Hadoop Lz4Codec +* [PARQUET-1904](https://issues.apache.org/jira/browse/PARQUET-1904) - [C++] Export file\_offset in RowGroupMetaData + + + # Apache Arrow 1.0.0 (2020-07-20) ## Bug Fixes diff --git a/LICENSE.txt b/LICENSE.txt index 673b146c807c..8a728a982185 100644 --- a/LICENSE.txt +++ b/LICENSE.txt @@ -849,9 +849,9 @@ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -------------------------------------------------------------------------------- -The files in cpp/src/arrow/vendored/utf8cpp/ have the following license +The files in cpp/src/arrow/vendored/utfcpp/ have the following license -Copyright 2006 Nemanja Trifunovic +Copyright 2006-2018 Nemanja Trifunovic Permission is hereby granted, free of charge, to any person or organization obtaining a copy of the software and accompanying documentation covered by @@ -2207,3 +2207,19 @@ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +-------------------------------------------------------------------------------- + +The files in cpp/src/arrow/vendored/portable-snippets/ contain code from + +https://github.com/nemequ/portable-snippets + +and have the following copyright notice: + +Each source file contains a preamble explaining the license situation +for that file, which takes priority over this file. With the +exception of some code pulled in from other repositories (such as +µnit, an MIT-licensed project which is used for testing), the code is +public domain, released using the CC0 1.0 Universal dedication (*). + +(*) https://creativecommons.org/publicdomain/zero/1.0/legalcode diff --git a/README.md b/README.md index b1e96dccc6b0..133018c72dfc 100644 --- a/README.md +++ b/README.md @@ -36,7 +36,7 @@ Major components of the project include: a standard and efficient in-memory representation of various datatypes, plain or nested - [The Arrow IPC Format](https://github.com/apache/arrow/blob/master/docs/source/format/Columnar.rst#serialization-and-interprocess-communication-ipc): an efficient serialization of the Arrow format and associated metadata, - for communication between processes and heterogenous environments + for communication between processes and heterogeneous environments - [The Arrow Flight RPC protocol](https://github.com/apache/arrow/tree/master/format/Flight.proto): based on the Arrow IPC format, a building block for remote services exchanging Arrow data with application-defined semantics (for example a storage server or a database) @@ -60,7 +60,7 @@ Arrow is an [Apache Software Foundation](https://www.apache.org) project. Learn ## What's in the Arrow libraries? -The reference Arrow libraries contain a number of distinct software components: +The reference Arrow libraries contain many distinct software components: - Columnar vector and table-like containers (similar to data frames) supporting flat or nested types diff --git a/c_glib/arrow-glib/codec.cpp b/c_glib/arrow-glib/codec.cpp index fdd61e70a177..33b3d1c9149e 100644 --- a/c_glib/arrow-glib/codec.cpp +++ b/c_glib/arrow-glib/codec.cpp @@ -38,7 +38,7 @@ G_BEGIN_DECLS */ typedef struct GArrowCodecPrivate_ { - arrow::util::Codec *codec; + std::shared_ptr codec; } GArrowCodecPrivate; enum { @@ -57,7 +57,7 @@ garrow_codec_finalize(GObject *object) { auto priv = GARROW_CODEC_GET_PRIVATE(object); - delete priv->codec; + priv->codec.~shared_ptr(); G_OBJECT_CLASS(garrow_codec_parent_class)->finalize(object); } @@ -72,7 +72,8 @@ garrow_codec_set_property(GObject *object, switch (prop_id) { case PROP_CODEC: - priv->codec = static_cast(g_value_get_pointer(value)); + priv->codec = + *static_cast *>(g_value_get_pointer(value)); break; default: G_OBJECT_WARN_INVALID_PROPERTY_ID(object, prop_id, pspec); @@ -96,6 +97,8 @@ garrow_codec_get_property(GObject *object, static void garrow_codec_init(GArrowCodec *object) { + auto priv = GARROW_CODEC_GET_PRIVATE(object); + new(&priv->codec) std::shared_ptr; } static void @@ -111,7 +114,7 @@ garrow_codec_class_init(GArrowCodecClass *klass) spec = g_param_spec_pointer("codec", "Codec", - "The raw arrow::util::Codec *", + "The raw std::shared_ptr *", static_cast(G_PARAM_WRITABLE | G_PARAM_CONSTRUCT_ONLY)); g_object_class_install_property(gobject_class, PROP_CODEC, spec); @@ -133,7 +136,9 @@ garrow_codec_new(GArrowCompressionType type, auto arrow_type = garrow_compression_type_to_raw(type); auto arrow_codec = arrow::util::Codec::Create(arrow_type); if (garrow::check(error, arrow_codec, "[codec][new]")) { - return garrow_codec_new_raw(arrow_codec.ValueOrDie().release()); + std::shared_ptr arrow_codec_shared = + std::move(*arrow_codec); + return garrow_codec_new_raw(&arrow_codec_shared); } else { return NULL; } @@ -151,7 +156,46 @@ const gchar * garrow_codec_get_name(GArrowCodec *codec) { auto arrow_codec = garrow_codec_get_raw(codec); - return arrow_codec->name(); + if (!arrow_codec) { + return NULL; + } + return arrow_codec->name().c_str(); +} + +/** + * garrow_codec_get_compression_type: + * @codec: A #GArrowCodec. + * + * Returns: The compression type of the codec. + * + * Since: 2.0.0 + */ +GArrowCompressionType +garrow_codec_get_compression_type(GArrowCodec *codec) +{ + auto arrow_codec = garrow_codec_get_raw(codec); + if (!arrow_codec) { + return GARROW_COMPRESSION_TYPE_UNCOMPRESSED; + } + return garrow_compression_type_from_raw(arrow_codec->compression_type()); +} + +/** + * garrow_codec_get_compression_level: + * @codec: A #GArrowCodec. + * + * Returns: The compression level of the codec. + * + * Since: 2.0.0 + */ +gint +garrow_codec_get_compression_level(GArrowCodec *codec) +{ + auto arrow_codec = garrow_codec_get_raw(codec); + if (!arrow_codec) { + return arrow::util::Codec::UseDefaultCompressionLevel(); + } + return arrow_codec->compression_level(); } G_END_DECLS @@ -207,7 +251,7 @@ garrow_compression_type_to_raw(GArrowCompressionType type) } GArrowCodec * -garrow_codec_new_raw(arrow::util::Codec *arrow_codec) +garrow_codec_new_raw(std::shared_ptr *arrow_codec) { auto codec = GARROW_CODEC(g_object_new(GARROW_TYPE_CODEC, "codec", arrow_codec, @@ -215,7 +259,7 @@ garrow_codec_new_raw(arrow::util::Codec *arrow_codec) return codec; } -arrow::util::Codec * +std::shared_ptr garrow_codec_get_raw(GArrowCodec *codec) { auto priv = GARROW_CODEC_GET_PRIVATE(codec); diff --git a/c_glib/arrow-glib/codec.h b/c_glib/arrow-glib/codec.h index 5feab2b7d4d1..6e177af9eede 100644 --- a/c_glib/arrow-glib/codec.h +++ b/c_glib/arrow-glib/codec.h @@ -20,6 +20,7 @@ #pragma once #include +#include G_BEGIN_DECLS @@ -63,5 +64,11 @@ GArrowCodec *garrow_codec_new(GArrowCompressionType type, GError **error); const gchar *garrow_codec_get_name(GArrowCodec *codec); +GARROW_AVAILABLE_IN_2_0 +GArrowCompressionType +garrow_codec_get_compression_type(GArrowCodec *codec); +GARROW_AVAILABLE_IN_2_0 +gint +garrow_codec_get_compression_level(GArrowCodec *codec); G_END_DECLS diff --git a/c_glib/arrow-glib/codec.hpp b/c_glib/arrow-glib/codec.hpp index 14c3ad77ccf1..f4cfaba18a00 100644 --- a/c_glib/arrow-glib/codec.hpp +++ b/c_glib/arrow-glib/codec.hpp @@ -29,6 +29,6 @@ arrow::Compression::type garrow_compression_type_to_raw(GArrowCompressionType type); GArrowCodec * -garrow_codec_new_raw(arrow::util::Codec *arrow_codec); -arrow::util::Codec * +garrow_codec_new_raw(std::shared_ptr *arrow_codec); +std::shared_ptr garrow_codec_get_raw(GArrowCodec *codec); diff --git a/c_glib/arrow-glib/composite-array.cpp b/c_glib/arrow-glib/composite-array.cpp index 14dda3735754..688c548bf2ff 100644 --- a/c_glib/arrow-glib/composite-array.cpp +++ b/c_glib/arrow-glib/composite-array.cpp @@ -140,6 +140,53 @@ garrow_base_list_array_get_value(GArrowArray *array, "parent", array, NULL); }; + +template +GArrowArray * +garrow_base_list_array_get_values(GArrowArray *array) +{ + auto arrow_array = garrow_array_get_raw(array); + auto arrow_list_array = + std::static_pointer_cast(arrow_array); + auto arrow_values = arrow_list_array->values(); + return garrow_array_new_raw(&arrow_values, + "array", &arrow_values, + "parent", array, + NULL); +}; + +template +typename LIST_ARRAY_CLASS::offset_type +garrow_base_list_array_get_value_offset(GArrowArray *array, gint64 i) +{ + auto arrow_array = garrow_array_get_raw(array); + auto arrow_list_array = + std::static_pointer_cast(arrow_array); + return arrow_list_array->value_offset(i); +}; + +template +typename LIST_ARRAY_CLASS::offset_type +garrow_base_list_array_get_value_length(GArrowArray *array, gint64 i) +{ + auto arrow_array = garrow_array_get_raw(array); + auto arrow_list_array = + std::static_pointer_cast(arrow_array); + return arrow_list_array->value_length(i); +}; + +template +const typename LIST_ARRAY_CLASS::offset_type * +garrow_base_list_array_get_value_offsets(GArrowArray *array, gint64 *n_offsets) +{ + auto arrow_array = garrow_array_get_raw(array); + *n_offsets = arrow_array->length() + 1; + auto arrow_list_array = + std::static_pointer_cast(arrow_array); + return arrow_list_array->raw_value_offsets(); +}; + + G_BEGIN_DECLS static void @@ -279,6 +326,70 @@ garrow_list_array_get_value(GArrowListArray *array, GARROW_ARRAY(array), i); } +/** + * garrow_list_array_get_values: + * @array: A #GArrowListArray. + * + * Returns: (transfer full): The array containing the list's values. + * + * Since: 2.0.0 + */ +GArrowArray * +garrow_list_array_get_values(GArrowListArray *array) +{ + return garrow_base_list_array_get_values( + GARROW_ARRAY(array)); +} + +/** + * garrow_list_array_get_offset: + * @array: A #GArrowListArray. + * @i: The index of the offset of the target value. + * + * Returns: The target offset in the array containing the list's values. + * + * Since: 2.0.0 + */ +gint32 +garrow_list_array_get_value_offset(GArrowListArray *array, gint64 i) +{ + return garrow_base_list_array_get_value_offset( + GARROW_ARRAY(array), i); +} + +/** + * garrow_list_array_get_value_length: + * @array: A #GArrowListArray. + * @i: The index of the length of the target value. + * + * Returns: The target length in the array containing the list's values. + * + * Since: 2.0.0 + */ +gint32 +garrow_list_array_get_value_length(GArrowListArray *array, gint64 i) +{ + return garrow_base_list_array_get_value_length( + GARROW_ARRAY(array), i); +} + +/** + * garrow_list_array_get_value_offsets: + * @array: A #GArrowListArray. + * @n_offsets: The number of offsets to be returned. + * + * Returns: (array length=n_offsets): The target offsets in the array + * containing the list's values. + * + * Since: 2.0.0 + */ +const gint32 * +garrow_list_array_get_value_offsets(GArrowListArray *array, gint64 *n_offsets) +{ + return garrow_base_list_array_get_value_offsets( + GARROW_ARRAY(array), n_offsets); +} + typedef struct GArrowLargeListArrayPrivate_ { GArrowArray *raw_values; @@ -434,6 +545,71 @@ garrow_large_list_array_get_value(GArrowLargeListArray *array, i); } +/** + * garrow_large_list_array_get_values: + * @array: A #GArrowLargeListArray. + * + * Returns: (transfer full): The array containing the list's values. + * + * Since: 2.0.0 + */ +GArrowArray * +garrow_large_list_array_get_values(GArrowLargeListArray *array) +{ + return garrow_base_list_array_get_values( + GARROW_ARRAY(array)); +} + +/** + * garrow_large_list_array_get_value_offset: + * @array: A #GArrowLargeListArray. + * @i: The index of the offset of the target value. + * + * Returns: The target offset in the array containing the list's values. + * + * Since: 2.0.0 + */ +gint64 +garrow_large_list_array_get_value_offset(GArrowLargeListArray *array, gint64 i) +{ + return garrow_base_list_array_get_value_offset( + GARROW_ARRAY(array), i); +} + +/** + * garrow_large_list_array_get_length: + * @array: A #GArrowLargeListArray. + * @i: The index of the length of the target value. + * + * Returns: The target length in the array containing the list's values. + * + * Since: 2.0.0 + */ +gint64 +garrow_large_list_array_get_value_length(GArrowLargeListArray *array, gint64 i) +{ + return garrow_base_list_array_get_value_length( + GARROW_ARRAY(array), i); +} + +/** + * garrow_large_list_array_get_value_offsets: + * @array: A #GArrowLargeListArray. + * @n_offsets: The number of offsets to be returned. + * + * Returns: (array length=n_offsets): The target offsets in the array + * containing the list's values. + * + * Since: 2.0.0 + */ +const gint64 * +garrow_large_list_array_get_value_offsets(GArrowLargeListArray *array, + gint64 *n_offsets) +{ + return garrow_base_list_array_get_value_offsets( + GARROW_ARRAY(array), n_offsets); +} + typedef struct GArrowStructArrayPrivate_ { GPtrArray *fields; diff --git a/c_glib/arrow-glib/composite-array.h b/c_glib/arrow-glib/composite-array.h index dd0c72668f97..cfaeb4c768c4 100644 --- a/c_glib/arrow-glib/composite-array.h +++ b/c_glib/arrow-glib/composite-array.h @@ -47,6 +47,18 @@ GArrowListArray *garrow_list_array_new(GArrowDataType *data_type, GArrowDataType *garrow_list_array_get_value_type(GArrowListArray *array); GArrowArray *garrow_list_array_get_value(GArrowListArray *array, gint64 i); +GARROW_AVAILABLE_IN_2_0 +GArrowArray *garrow_list_array_get_values(GArrowListArray *array); +GARROW_AVAILABLE_IN_2_0 +gint32 garrow_list_array_get_value_offset(GArrowListArray *array, + gint64 i); +GARROW_AVAILABLE_IN_2_0 +gint32 garrow_list_array_get_value_length(GArrowListArray *array, + gint64 i); +GARROW_AVAILABLE_IN_2_0 +const gint32 * +garrow_list_array_get_value_offsets(GArrowListArray *array, + gint64 *n_offsets); #define GARROW_TYPE_LARGE_LIST_ARRAY (garrow_large_list_array_get_type()) @@ -73,6 +85,18 @@ GArrowDataType *garrow_large_list_array_get_value_type(GArrowLargeListArray *arr GARROW_AVAILABLE_IN_0_16 GArrowArray *garrow_large_list_array_get_value(GArrowLargeListArray *array, gint64 i); +GARROW_AVAILABLE_IN_2_0 +GArrowArray *garrow_large_list_array_get_values(GArrowLargeListArray *array); +GARROW_AVAILABLE_IN_2_0 +gint64 garrow_large_list_array_get_value_offset(GArrowLargeListArray *array, + gint64 i); +GARROW_AVAILABLE_IN_2_0 +gint64 garrow_large_list_array_get_value_length(GArrowLargeListArray *array, + gint64 i); +GARROW_AVAILABLE_IN_2_0 +const gint64 * +garrow_large_list_array_get_value_offsets(GArrowLargeListArray *array, + gint64 *n_offsets); #define GARROW_TYPE_STRUCT_ARRAY (garrow_struct_array_get_type()) diff --git a/c_glib/arrow-glib/compute.cpp b/c_glib/arrow-glib/compute.cpp index 3e318999d31d..777adee41a5b 100644 --- a/c_glib/arrow-glib/compute.cpp +++ b/c_glib/arrow-glib/compute.cpp @@ -300,7 +300,7 @@ garrow_function_find(const gchar *name) * @error: (nullable): Return location for a #GError or %NULL. * * Returns: (nullable) (transfer full): - * A return value of the execution as #GArrowData on success, %NULL on error. + * A return value of the execution as #GArrowDatum on success, %NULL on error. * * Since: 1.0.0 */ @@ -1246,7 +1246,8 @@ garrow_array_count_values(GArrowArray *array, auto arrow_array = garrow_array_get_raw(array); auto arrow_counted_values = arrow::compute::ValueCounts(arrow_array); if (garrow::check(error, arrow_counted_values, "[array][count-values]")) { - return GARROW_STRUCT_ARRAY(garrow_array_new_raw(&(*arrow_counted_values))); + std::shared_ptr arrow_counted_values_array = *arrow_counted_values; + return GARROW_STRUCT_ARRAY(garrow_array_new_raw(&arrow_counted_values_array)); } else { return NULL; } diff --git a/c_glib/arrow-glib/input-stream.cpp b/c_glib/arrow-glib/input-stream.cpp index 3751d41ad3ab..84904b742658 100644 --- a/c_glib/arrow-glib/input-stream.cpp +++ b/c_glib/arrow-glib/input-stream.cpp @@ -1132,7 +1132,7 @@ garrow_compressed_input_stream_new(GArrowCodec *codec, GArrowInputStream *raw, GError **error) { - auto arrow_codec = garrow_codec_get_raw(codec); + auto arrow_codec = garrow_codec_get_raw(codec).get(); auto arrow_raw = garrow_input_stream_get_raw(raw); auto arrow_stream = arrow::io::CompressedInputStream::Make(arrow_codec, arrow_raw); diff --git a/c_glib/arrow-glib/ipc-options.cpp b/c_glib/arrow-glib/ipc-options.cpp index 1cddd25bb6de..b9b2c4143486 100644 --- a/c_glib/arrow-glib/ipc-options.cpp +++ b/c_glib/arrow-glib/ipc-options.cpp @@ -21,6 +21,7 @@ # include #endif +#include #include #include @@ -242,6 +243,7 @@ garrow_read_options_set_included_fields(GArrowReadOptions *options, typedef struct GArrowWriteOptionsPrivate_ { arrow::ipc::IpcWriteOptions options; + GArrowCodec *codec; } GArrowWriteOptionsPrivate; enum { @@ -249,8 +251,7 @@ enum { PROP_WRITE_OPTIONS_MAX_RECURSION_DEPTH, PROP_WRITE_OPTIONS_ALIGNMENT, PROP_WRITE_OPTIONS_WRITE_LEGACY_IPC_FORMAT, - PROP_WRITE_OPTIONS_COMPRESSION, - PROP_WRITE_OPTIONS_COMPRESSION_LEVEL, + PROP_WRITE_OPTIONS_CODEC, PROP_WRITE_OPTIONS_USE_THREADS, }; @@ -263,6 +264,19 @@ G_DEFINE_TYPE_WITH_PRIVATE(GArrowWriteOptions, garrow_write_options_get_instance_private( \ GARROW_WRITE_OPTIONS(obj))) +static void +garrow_write_options_dispose(GObject *object) +{ + auto priv = GARROW_WRITE_OPTIONS_GET_PRIVATE(object); + + if (priv->codec) { + g_object_unref(priv->codec); + priv->codec = NULL; + } + + G_OBJECT_CLASS(garrow_write_options_parent_class)->dispose(object); +} + static void garrow_write_options_finalize(GObject *object) { @@ -294,12 +308,12 @@ garrow_write_options_set_property(GObject *object, case PROP_WRITE_OPTIONS_WRITE_LEGACY_IPC_FORMAT: priv->options.write_legacy_ipc_format = g_value_get_boolean(value); break; - case PROP_WRITE_OPTIONS_COMPRESSION: - priv->options.compression = - static_cast(g_value_get_enum(value)); - break; - case PROP_WRITE_OPTIONS_COMPRESSION_LEVEL: - priv->options.compression_level = g_value_get_int(value); + case PROP_WRITE_OPTIONS_CODEC: + if (priv->codec) { + g_object_unref(priv->codec); + } + priv->codec = GARROW_CODEC(g_value_dup_object(value)); + priv->options.codec = garrow_codec_get_raw(priv->codec); break; case PROP_WRITE_OPTIONS_USE_THREADS: priv->options.use_threads = g_value_get_boolean(value); @@ -331,11 +345,8 @@ garrow_write_options_get_property(GObject *object, case PROP_WRITE_OPTIONS_WRITE_LEGACY_IPC_FORMAT: g_value_set_boolean(value, priv->options.write_legacy_ipc_format); break; - case PROP_WRITE_OPTIONS_COMPRESSION: - g_value_set_enum(value, priv->options.compression); - break; - case PROP_WRITE_OPTIONS_COMPRESSION_LEVEL: - g_value_set_int(value, priv->options.compression_level); + case PROP_WRITE_OPTIONS_CODEC: + g_value_set_object(value, priv->codec); break; case PROP_WRITE_OPTIONS_USE_THREADS: g_value_set_boolean(value, priv->options.use_threads); @@ -352,6 +363,11 @@ garrow_write_options_init(GArrowWriteOptions *object) auto priv = GARROW_WRITE_OPTIONS_GET_PRIVATE(object); new(&priv->options) arrow::ipc::IpcWriteOptions; priv->options = arrow::ipc::IpcWriteOptions::Defaults(); + if (priv->options.codec) { + priv->codec = garrow_codec_new_raw(&(priv->options.codec)); + } else { + priv->codec = NULL; + } } static void @@ -359,6 +375,7 @@ garrow_write_options_class_init(GArrowWriteOptionsClass *klass) { auto gobject_class = G_OBJECT_CLASS(klass); + gobject_class->dispose = garrow_write_options_dispose; gobject_class->finalize = garrow_write_options_finalize; gobject_class->set_property = garrow_write_options_set_property; gobject_class->get_property = garrow_write_options_get_property; @@ -441,42 +458,24 @@ garrow_write_options_class_init(GArrowWriteOptionsClass *klass) spec); /** - * GArrowWriteOptions:compression: + * GArrowWriteOptions:codec: * * Codec to use for compressing and decompressing record batch body * buffers. This is not part of the Arrow IPC protocol and only for - * internal use (e.g. Feather files). May only be LZ4_FRAME and - * ZSTD. + * internal use (e.g. Feather files). * - * Since: 1.0.0 - */ - spec = g_param_spec_enum("compression", - "Compression", - "Codec to use for " - "compressing record batch body buffers.", - GARROW_TYPE_COMPRESSION_TYPE, - options.compression, - static_cast(G_PARAM_READWRITE)); - g_object_class_install_property(gobject_class, - PROP_WRITE_OPTIONS_COMPRESSION, - spec); - - /** - * GArrowWriteOptions:compression-level: - * - * The level for compression. + * May only be UNCOMPRESSED, LZ4_FRAME and ZSTD. * - * Since: 1.0.0 + * Since: 2.0.0 */ - spec = g_param_spec_int("compression-level", - "Compression level", - "The level for compression", - G_MININT, - G_MAXINT, - options.compression_level, - static_cast(G_PARAM_READWRITE)); + spec = g_param_spec_object("codec", + "Codec", + "Codec to use for " + "compressing record batch body buffers.", + GARROW_TYPE_CODEC, + static_cast(G_PARAM_READWRITE)); g_object_class_install_property(gobject_class, - PROP_WRITE_OPTIONS_COMPRESSION_LEVEL, + PROP_WRITE_OPTIONS_CODEC, spec); /** diff --git a/c_glib/arrow-glib/output-stream.cpp b/c_glib/arrow-glib/output-stream.cpp index 2c3ccafdb138..1619bac45d4d 100644 --- a/c_glib/arrow-glib/output-stream.cpp +++ b/c_glib/arrow-glib/output-stream.cpp @@ -688,7 +688,7 @@ garrow_compressed_output_stream_new(GArrowCodec *codec, GArrowOutputStream *raw, GError **error) { - auto arrow_codec = garrow_codec_get_raw(codec); + auto arrow_codec = garrow_codec_get_raw(codec).get(); auto arrow_raw = garrow_output_stream_get_raw(raw); auto arrow_stream = arrow::io::CompressedOutputStream::Make(arrow_codec, arrow_raw); diff --git a/c_glib/arrow-glib/reader.cpp b/c_glib/arrow-glib/reader.cpp index 51ad3ac444d6..f2202515965e 100644 --- a/c_glib/arrow-glib/reader.cpp +++ b/c_glib/arrow-glib/reader.cpp @@ -332,7 +332,7 @@ garrow_record_batch_file_reader_finalize(GObject *object) { auto priv = GARROW_RECORD_BATCH_FILE_READER_GET_PRIVATE(object); - priv->record_batch_file_reader = nullptr; + priv->record_batch_file_reader.~shared_ptr(); G_OBJECT_CLASS(garrow_record_batch_file_reader_parent_class)->finalize(object); } @@ -372,6 +372,9 @@ garrow_record_batch_file_reader_get_property(GObject *object, static void garrow_record_batch_file_reader_init(GArrowRecordBatchFileReader *object) { + auto priv = GARROW_RECORD_BATCH_FILE_READER_GET_PRIVATE(object); + new(&priv->record_batch_file_reader) + std::shared_ptr; } static void diff --git a/c_glib/arrow-glib/writer.cpp b/c_glib/arrow-glib/writer.cpp index 074c83af1208..82d18e58dbd8 100644 --- a/c_glib/arrow-glib/writer.cpp +++ b/c_glib/arrow-glib/writer.cpp @@ -235,10 +235,10 @@ garrow_record_batch_stream_writer_new(GArrowOutputStream *sink, GArrowSchema *schema, GError **error) { - auto arrow_sink = garrow_output_stream_get_raw(sink).get(); + auto arrow_sink = garrow_output_stream_get_raw(sink); auto arrow_schema = garrow_schema_get_raw(schema); auto arrow_writer_result = - arrow::ipc::NewStreamWriter(arrow_sink, arrow_schema); + arrow::ipc::MakeStreamWriter(arrow_sink, arrow_schema); if (garrow::check(error, arrow_writer_result, "[record-batch-stream-writer][open]")) { @@ -280,11 +280,11 @@ garrow_record_batch_file_writer_new(GArrowOutputStream *sink, GArrowSchema *schema, GError **error) { - auto arrow_sink = garrow_output_stream_get_raw(sink).get(); + auto arrow_sink = garrow_output_stream_get_raw(sink); auto arrow_schema = garrow_schema_get_raw(schema); std::shared_ptr arrow_writer; auto arrow_writer_result = - arrow::ipc::NewFileWriter(arrow_sink, arrow_schema); + arrow::ipc::MakeFileWriter(arrow_sink, arrow_schema); if (garrow::check(error, arrow_writer_result, "[record-batch-file-writer][open]")) { diff --git a/c_glib/configure.ac b/c_glib/configure.ac index 5919ec95d7c7..f7e2879c58e2 100644 --- a/c_glib/configure.ac +++ b/c_glib/configure.ac @@ -17,7 +17,7 @@ AC_PREREQ(2.65) -m4_define([arrow_glib_version], 2.0.0-SNAPSHOT) +m4_define([arrow_glib_version], 2.0.0) AC_INIT([arrow-glib], arrow_glib_version, [https://issues.apache.org/jira/browse/ARROW], diff --git a/c_glib/doc/arrow-glib/arrow-glib-docs.xml b/c_glib/doc/arrow-glib/arrow-glib-docs.xml index 57a53c2eec7b..72a01f50e4f5 100644 --- a/c_glib/doc/arrow-glib/arrow-glib-docs.xml +++ b/c_glib/doc/arrow-glib/arrow-glib-docs.xml @@ -179,6 +179,10 @@ Index of deprecated API + + Index of new symbols in 2.0.0 + + Index of new symbols in 1.0.0 diff --git a/c_glib/meson.build b/c_glib/meson.build index 7b8b21c7b20f..bf509300ff5f 100644 --- a/c_glib/meson.build +++ b/c_glib/meson.build @@ -23,7 +23,7 @@ project('arrow-glib', 'c', 'cpp', 'cpp_std=c++11', ]) -version = '2.0.0-SNAPSHOT' +version = '2.0.0' if version.endswith('-SNAPSHOT') version_numbers = version.split('-')[0].split('.') version_tag = version.split('-')[1] diff --git a/c_glib/test/dataset/test-scan-options.rb b/c_glib/test/dataset/test-scan-options.rb index c5657fe7e175..1f5b77f2e9ff 100644 --- a/c_glib/test/dataset/test-scan-options.rb +++ b/c_glib/test/dataset/test-scan-options.rb @@ -28,7 +28,7 @@ def test_schema end def test_batch_size - assert_equal(1<<15, + assert_equal(1<<20, @scan_options.batch_size) @scan_options.batch_size = 42 assert_equal(42, diff --git a/c_glib/test/test-codec.rb b/c_glib/test/test-codec.rb index 6617815df9b8..a32ec4dc7579 100644 --- a/c_glib/test/test-codec.rb +++ b/c_glib/test/test-codec.rb @@ -20,4 +20,14 @@ def test_name codec = Arrow::Codec.new(:gzip) assert_equal("gzip", codec.name) end + + def test_compression_type + codec = Arrow::Codec.new(:gzip) + assert_equal(Arrow::CompressionType::GZIP, codec.compression_type) + end + + def test_compression_level + codec = Arrow::Codec.new(:gzip) + assert_equal(9, codec.compression_level) + end end diff --git a/c_glib/test/test-large-list-array.rb b/c_glib/test/test-large-list-array.rb index 9840989ab891..2f7efab5a074 100644 --- a/c_glib/test/test-large-list-array.rb +++ b/c_glib/test/test-large-list-array.rb @@ -36,21 +36,11 @@ def test_new end def test_value - field = Arrow::Field.new("value", Arrow::Int64DataType.new) - data_type = Arrow::LargeListDataType.new(field) - builder = Arrow::LargeListArrayBuilder.new(data_type) - value_builder = builder.value_builder - - builder.append_value - value_builder.append_value(-29) - value_builder.append_value(29) - - builder.append_value - value_builder.append_value(-1) - value_builder.append_value(0) - value_builder.append_value(1) - - array = builder.finish + array = build_large_list_array(Arrow::Int8DataType.new, + [ + [-29, 29], + [-1, 0, 1], + ]) value = array.get_value(1) assert_equal([-1, 0, 1], value.length.times.collect {|i| value.get_value(i)}) @@ -63,4 +53,46 @@ def test_value_type array = builder.finish assert_equal(Arrow::Int64DataType.new, array.value_type) end + + + def test_values + array = build_large_list_array(Arrow::Int8DataType.new, + [ + [-29, 29], + [-1, 0, 1], + ]) + values = array.values + assert_equal([-29, 29, -1, 0, 1], + values.length.times.collect {|i| values.get_value(i)}) + end + + def test_value_offset + array = build_large_list_array(Arrow::Int8DataType.new, + [ + [-29, 29], + [-1, 0, 1], + ]) + assert_equal([0, 2], + array.length.times.collect {|i| array.get_value_offset(i)}) + end + + def test_value_length + array = build_large_list_array(Arrow::Int8DataType.new, + [ + [-29, 29], + [-1, 0, 1], + ]) + assert_equal([2, 3], + array.length.times.collect {|i| array.get_value_length(i)}) + end + + def test_value_offsets + array = build_large_list_array(Arrow::Int8DataType.new, + [ + [-29, 29], + [-1, 0, 1], + ]) + assert_equal([0, 2, 5], + array.value_offsets) + end end diff --git a/c_glib/test/test-list-array.rb b/c_glib/test/test-list-array.rb index eea16ccbbb35..f94b28dd1cd6 100644 --- a/c_glib/test/test-list-array.rb +++ b/c_glib/test/test-list-array.rb @@ -36,21 +36,11 @@ def test_new end def test_value - field = Arrow::Field.new("value", Arrow::Int8DataType.new) - data_type = Arrow::ListDataType.new(field) - builder = Arrow::ListArrayBuilder.new(data_type) - value_builder = builder.value_builder - - builder.append_value - value_builder.append_value(-29) - value_builder.append_value(29) - - builder.append_value - value_builder.append_value(-1) - value_builder.append_value(0) - value_builder.append_value(1) - - array = builder.finish + array = build_list_array(Arrow::Int8DataType.new, + [ + [-29, 29], + [-1, 0, 1], + ]) value = array.get_value(1) assert_equal([-1, 0, 1], value.length.times.collect {|i| value.get_value(i)}) @@ -63,4 +53,45 @@ def test_value_type array = builder.finish assert_equal(Arrow::Int8DataType.new, array.value_type) end + + def test_values + array = build_list_array(Arrow::Int8DataType.new, + [ + [-29, 29], + [-1, 0, 1], + ]) + values = array.values + assert_equal([-29, 29, -1, 0, 1], + values.length.times.collect {|i| values.get_value(i)}) + end + + def test_value_offset + array = build_list_array(Arrow::Int8DataType.new, + [ + [-29, 29], + [-1, 0, 1], + ]) + assert_equal([0, 2], + array.length.times.collect {|i| array.get_value_offset(i)}) + end + + def test_value_length + array = build_list_array(Arrow::Int8DataType.new, + [ + [-29, 29], + [-1, 0, 1], + ]) + assert_equal([2, 3], + array.length.times.collect {|i| array.get_value_length(i)}) + end + + def test_value_offsets + array = build_list_array(Arrow::Int8DataType.new, + [ + [-29, 29], + [-1, 0, 1], + ]) + assert_equal([0, 2, 5], + array.value_offsets) + end end diff --git a/c_glib/test/test-write-options.rb b/c_glib/test/test-write-options.rb index d30b78b9cdbf..c528ce673d45 100644 --- a/c_glib/test/test-write-options.rb +++ b/c_glib/test/test-write-options.rb @@ -73,27 +73,15 @@ def test_accessor end end - sub_test_case("compression") do + sub_test_case("codec") do def test_default - assert_equal(Arrow::CompressionType::UNCOMPRESSED, - @options.compression) + assert_nil(@options.codec) end def test_accessor - @options.compression = :zstd - assert_equal(Arrow::CompressionType::ZSTD, - @options.compression) - end - end - - sub_test_case("compression-level") do - def test_default - assert_equal(-(2 ** 31), @options.compression_level) - end - - def test_accessor - @options.compression_level = 8 - assert_equal(8, @options.compression_level) + @options.codec = Arrow::Codec.new(:zstd) + assert_equal("zstd", + @options.codec.name) end end diff --git a/ci/appveyor-cpp-build.bat b/ci/appveyor-cpp-build.bat index a2ce0765617b..6b9309396604 100644 --- a/ci/appveyor-cpp-build.bat +++ b/ci/appveyor-cpp-build.bat @@ -89,7 +89,7 @@ pushd cpp\build @rem and enable runtime assertions. cmake -G "%GENERATOR%" %CMAKE_ARGS% ^ - -DARROW_BOOST_USE_SHARED=OFF ^ + -DARROW_BOOST_USE_SHARED=ON ^ -DARROW_BUILD_EXAMPLES=ON ^ -DARROW_BUILD_STATIC=OFF ^ -DARROW_BUILD_TESTS=ON ^ diff --git a/ci/appveyor-cpp-setup.bat b/ci/appveyor-cpp-setup.bat index 14bc0fd7a215..809907a3d1c9 100644 --- a/ci/appveyor-cpp-setup.bat +++ b/ci/appveyor-cpp-setup.bat @@ -62,7 +62,6 @@ if "%JOB%" NEQ "Build_Debug" ( --file=ci\conda_env_python.yml ^ %CONDA_PACKAGES% ^ "cmake=3.17" ^ - "boost-cpp<1.70" ^ "ninja" ^ "nomkl" ^ "pandas" ^ diff --git a/ci/conda_env_cpp.yml b/ci/conda_env_cpp.yml index a0c2e99aca7b..90cef3ea2d1c 100644 --- a/ci/conda_env_cpp.yml +++ b/ci/conda_env_cpp.yml @@ -25,7 +25,7 @@ cmake gflags glog gmock>=1.8.1 -grpc-cpp>=1.21.4 +grpc-cpp>=1.27.3 gtest=1.8.1 libprotobuf libutf8proc diff --git a/ci/conda_env_sphinx.yml b/ci/conda_env_sphinx.yml index 318ef75e46c3..8654d231065e 100644 --- a/ci/conda_env_sphinx.yml +++ b/ci/conda_env_sphinx.yml @@ -19,6 +19,6 @@ breathe doxygen ipython -# Pinned per ARROW-8340 -sphinx=2.4.4 +# Pinned per ARROW-9693 +sphinx=3.1.2 sphinx_rtd_theme diff --git a/ci/docker/conda-integration.dockerfile b/ci/docker/conda-integration.dockerfile index 088d93e39211..e6e5ac859dd1 100644 --- a/ci/docker/conda-integration.dockerfile +++ b/ci/docker/conda-integration.dockerfile @@ -21,7 +21,7 @@ FROM ${repo}:${arch}-conda-cpp ARG arch=amd64 ARG maven=3.5 -ARG node=11 +ARG node=14 ARG jdk=8 ARG go=1.12 diff --git a/ci/docker/conda-python-kartothek.dockerfile b/ci/docker/conda-python-kartothek.dockerfile index aa013fe5f3da..d523161822c0 100644 --- a/ci/docker/conda-python-kartothek.dockerfile +++ b/ci/docker/conda-python-kartothek.dockerfile @@ -22,11 +22,17 @@ FROM ${repo}:${arch}-conda-python-${python} # install kartothek dependencies from conda-forge RUN conda install -c conda-forge -q \ + attrs \ + click \ + cloudpickle \ dask \ decorator \ + freezegun \ msgpack-python \ + prompt-toolkit \ pytest-mock \ pytest-xdist \ + pyyaml \ simplejson \ simplekv \ storefact \ diff --git a/ci/docker/conda-python-spark.dockerfile b/ci/docker/conda-python-spark.dockerfile index a20f1ff35217..d3f0a2245820 100644 --- a/ci/docker/conda-python-spark.dockerfile +++ b/ci/docker/conda-python-spark.dockerfile @@ -36,10 +36,6 @@ ARG spark=master COPY ci/scripts/install_spark.sh /arrow/ci/scripts/ RUN /arrow/ci/scripts/install_spark.sh ${spark} /spark -# patch spark to build with current Arrow Java -COPY ci/etc/integration_spark_ARROW-9438.patch /arrow/ci/etc/ -RUN patch -d /spark -p1 -i /arrow/ci/etc/integration_spark_ARROW-9438.patch - # build cpp with tests ENV CC=gcc \ CXX=g++ \ diff --git a/ci/docker/debian-10-cpp.dockerfile b/ci/docker/debian-10-cpp.dockerfile index f86c009b57b1..74143dcbfa4f 100644 --- a/ci/docker/debian-10-cpp.dockerfile +++ b/ci/docker/debian-10-cpp.dockerfile @@ -17,6 +17,7 @@ ARG arch=amd64 FROM ${arch}/debian:10 +ARG arch ENV DEBIAN_FRONTEND noninteractive @@ -26,7 +27,7 @@ RUN \ ARG llvm RUN apt-get update -y -q && \ - apt-get install -y -q --no-install-recommends \ + apt-get install -y -q --no-install-recommends \ apt-transport-https \ ca-certificates \ gnupg \ @@ -49,6 +50,7 @@ RUN apt-get update -y -q && \ libbrotli-dev \ libbz2-dev \ libc-ares-dev \ + libcurl4-openssl-dev \ libgflags-dev \ libgmock-dev \ libgoogle-glog-dev \ @@ -71,6 +73,10 @@ RUN apt-get update -y -q && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* +COPY ci/scripts/install_minio.sh \ + /arrow/ci/scripts/ +RUN /arrow/ci/scripts/install_minio.sh ${arch} linux latest /usr/local + ENV ARROW_BUILD_TESTS=ON \ ARROW_DEPENDENCY_SOURCE=SYSTEM \ ARROW_DATASET=ON \ @@ -80,6 +86,7 @@ ENV ARROW_BUILD_TESTS=ON \ ARROW_ORC=ON \ ARROW_PARQUET=ON \ ARROW_PLASMA=ON \ + ARROW_S3=ON \ ARROW_USE_CCACHE=ON \ ARROW_WITH_BROTLI=ON \ ARROW_WITH_BZ2=ON \ @@ -87,6 +94,7 @@ ENV ARROW_BUILD_TESTS=ON \ ARROW_WITH_SNAPPY=ON \ ARROW_WITH_ZLIB=ON \ ARROW_WITH_ZSTD=ON \ + AWSSDK_SOURCE=BUNDLED \ cares_SOURCE=BUNDLED \ CC=gcc \ CXX=g++ \ diff --git a/ci/docker/debian-10-js.dockerfile b/ci/docker/debian-10-js.dockerfile index b43c45abdf49..5bb31f2e32ef 100644 --- a/ci/docker/debian-10-js.dockerfile +++ b/ci/docker/debian-10-js.dockerfile @@ -16,7 +16,7 @@ # under the License. ARG arch=amd64 -ARG node=11 +ARG node=14 FROM ${arch}/node:${node} ENV NODE_NO_WARNINGS=1 diff --git a/ci/docker/debian-10-rust.dockerfile b/ci/docker/debian-10-rust.dockerfile index b23b03c9a1c6..9c9c9b510484 100644 --- a/ci/docker/debian-10-rust.dockerfile +++ b/ci/docker/debian-10-rust.dockerfile @@ -58,14 +58,18 @@ RUN mkdir \ /arrow/rust/benchmarks/src \ /arrow/rust/datafusion/src \ /arrow/rust/integration-testing/src \ - /arrow/rust/parquet/src && \ + /arrow/rust/parquet/src \ + /arrow/rust/parquet_derive/src \ + /arrow/rust/parquet_derive_test/src && \ touch \ /arrow/rust/arrow-flight/src/lib.rs \ /arrow/rust/arrow/src/lib.rs \ /arrow/rust/benchmarks/src/lib.rs \ /arrow/rust/datafusion/src/lib.rs \ /arrow/rust/integration-testing/src/lib.rs \ - /arrow/rust/parquet/src/lib.rs + /arrow/rust/parquet/src/lib.rs \ + /arrow/rust/parquet_derive/src/lib.rs \ + /arrow/rust/parquet_derive_test/src/lib.rs # Compile dependencies for the whole workspace RUN cd /arrow/rust && cargo build --workspace --lib --all-features diff --git a/ci/docker/fedora-32-cpp.dockerfile b/ci/docker/fedora-32-cpp.dockerfile index 535f8b4b7619..40fe4617b12c 100644 --- a/ci/docker/fedora-32-cpp.dockerfile +++ b/ci/docker/fedora-32-cpp.dockerfile @@ -17,10 +17,11 @@ ARG arch FROM ${arch}/fedora:32 +ARG arch # install dependencies RUN dnf update -y && \ - dnf install -y \ + dnf install -y \ autoconf \ boost-devel \ brotli-devel \ @@ -29,6 +30,7 @@ RUN dnf update -y && \ ccache \ clang-devel \ cmake \ + curl-devel \ flatbuffers-devel \ java-1.8.0-openjdk-devel \ java-1.8.0-openjdk-headless \ @@ -54,9 +56,14 @@ RUN dnf update -y && \ snappy-devel \ thrift-devel \ utf8proc-devel \ + wget \ which \ zlib-devel +COPY ci/scripts/install_minio.sh \ + /arrow/ci/scripts/ +RUN /arrow/ci/scripts/install_minio.sh ${arch} linux latest /usr/local + # * gRPC 1.26 in Fedora 32 may have a problem. arrow-flight-test is stuck. ENV ARROW_BUILD_TESTS=ON \ ARROW_DEPENDENCY_SOURCE=SYSTEM \ @@ -67,6 +74,7 @@ ENV ARROW_BUILD_TESTS=ON \ ARROW_HOME=/usr/local \ ARROW_ORC=ON \ ARROW_PARQUET=ON \ + ARROW_S3=ON \ ARROW_USE_CCACHE=ON \ ARROW_WITH_BROTLI=ON \ ARROW_WITH_BZ2=ON \ @@ -74,6 +82,7 @@ ENV ARROW_BUILD_TESTS=ON \ ARROW_WITH_SNAPPY=ON \ ARROW_WITH_ZLIB=ON \ ARROW_WITH_ZSTD=ON \ + AWSSDK_SOURCE=BUNDLED \ CC=gcc \ CXX=g++ \ gRPC_SOURCE=BUNDLED \ diff --git a/ci/docker/linux-apt-c-glib.dockerfile b/ci/docker/linux-apt-c-glib.dockerfile index 3d1658ff7b8c..dbd13045521e 100644 --- a/ci/docker/linux-apt-c-glib.dockerfile +++ b/ci/docker/linux-apt-c-glib.dockerfile @@ -54,7 +54,7 @@ RUN luarocks install lgi # ERROR: Command errored out with exit status 1: /usr/bin/python3 /usr/share/python-wheels/pep517-0.7.0-py2.py3-none-any.whl/pep517/_in_process.py get_requires_for_build_wheel /tmp/tmpsk4jveay Check the logs for full command output. RUN (python3 -m pip install meson || \ python3 -m pip install --no-use-pep517 meson) && \ - gem install bundler + gem install --no-document bundler COPY c_glib/Gemfile /arrow/c_glib/ RUN bundle install --gemfile /arrow/c_glib/Gemfile diff --git a/ci/docker/linux-apt-docs.dockerfile b/ci/docker/linux-apt-docs.dockerfile index d0d98d5cd308..ec474f998616 100644 --- a/ci/docker/linux-apt-docs.dockerfile +++ b/ci/docker/linux-apt-docs.dockerfile @@ -38,8 +38,11 @@ RUN apt-get update -y && \ gobject-introspection \ gtk-doc-tools \ libcurl4-openssl-dev \ + libfontconfig1-dev \ + libfribidi-dev \ libgirepository1.0-dev \ libglib2.0-doc \ + libharfbuzz-dev \ libtool \ libxml2-dev \ ninja-build \ @@ -62,23 +65,31 @@ RUN /arrow/ci/scripts/util_download_apache.sh \ ENV PATH=/opt/apache-maven-${maven}/bin:$PATH RUN mvn -version -ARG node=11 +ARG node=14 RUN wget -q -O - https://deb.nodesource.com/setup_${node}.x | bash - && \ apt-get install -y nodejs && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* +# Sphinx is pinned because of ARROW-9693 RUN pip install \ meson \ breathe \ ipython \ - sphinx \ + sphinx==3.1.2 \ sphinx_rtd_theme COPY c_glib/Gemfile /arrow/c_glib/ -RUN gem install bundler && \ +RUN gem install --no-document bundler && \ bundle install --gemfile /arrow/c_glib/Gemfile +# Ensure parallel R package installation, set CRAN repo mirror, +# and use pre-built binaries where possible +COPY ci/etc/rprofile /arrow/ci/etc/ +RUN cat /arrow/ci/etc/rprofile >> $(R RHOME)/etc/Rprofile.site +# Also ensure parallel compilation of C/C++ code +RUN echo "MAKEFLAGS=-j$(R -s -e 'cat(parallel::detectCores())')" >> $(R RHOME)/etc/Makeconf + COPY ci/scripts/r_deps.sh /arrow/ci/scripts/ COPY r/DESCRIPTION /arrow/r/ RUN /arrow/ci/scripts/r_deps.sh /arrow && \ diff --git a/ci/docker/linux-apt-r.dockerfile b/ci/docker/linux-apt-r.dockerfile index 85827358dfbf..f47044e334b9 100644 --- a/ci/docker/linux-apt-r.dockerfile +++ b/ci/docker/linux-apt-r.dockerfile @@ -17,6 +17,7 @@ ARG base FROM ${base} +ARG arch # Build R # [1] https://www.digitalocean.com/community/tutorials/how-to-install-r-on-ubuntu-18-04 @@ -70,6 +71,10 @@ COPY ci/scripts/r_deps.sh /arrow/ci/scripts/ COPY r/DESCRIPTION /arrow/r/ RUN /arrow/ci/scripts/r_deps.sh /arrow +COPY ci/scripts/install_minio.sh \ + /arrow/ci/scripts/ +RUN /arrow/ci/scripts/install_minio.sh ${arch} linux latest /usr/local + # Set up Python 3 and its dependencies RUN ln -s /usr/bin/python3 /usr/local/bin/python && \ ln -s /usr/bin/pip3 /usr/local/bin/pip @@ -89,6 +94,7 @@ ENV \ ARROW_PARQUET=ON \ ARROW_PLASMA=OFF \ ARROW_PYTHON=ON \ + ARROW_S3=ON \ ARROW_USE_CCACHE=ON \ ARROW_USE_GLOG=OFF \ LC_ALL=en_US.UTF-8 diff --git a/ci/docker/linux-r.dockerfile b/ci/docker/linux-r.dockerfile index 1d963a20d14e..5223d7aafa55 100644 --- a/ci/docker/linux-r.dockerfile +++ b/ci/docker/linux-r.dockerfile @@ -24,12 +24,16 @@ FROM ${base} ARG r_bin=R ENV R_BIN=${r_bin} +ARG r_dev=FALSE +ENV ARROW_R_DEV=${r_dev} + # Make sure R is on the path for the R-hub devel versions (where RPREFIX is set in its dockerfile) ENV PATH "${RPREFIX}/bin:${PATH}" # Patch up some of the docker images COPY ci/scripts/r_docker_configure.sh /arrow/ci/scripts/ COPY ci/etc/rprofile /arrow/ci/etc/ +COPY ci/scripts/install_minio.sh /arrow/ci/scripts/ RUN /arrow/ci/scripts/r_docker_configure.sh COPY ci/scripts/r_deps.sh /arrow/ci/scripts/ diff --git a/ci/docker/ubuntu-14.04-cpp.dockerfile b/ci/docker/ubuntu-14.04-cpp.dockerfile index 116973e6c804..6335dcc46e7b 100644 --- a/ci/docker/ubuntu-14.04-cpp.dockerfile +++ b/ci/docker/ubuntu-14.04-cpp.dockerfile @@ -91,4 +91,4 @@ ENV ARROW_BUILD_TESTS=ON \ RE2_SOURCE=BUNDLED \ Thrift_SOURCE=BUNDLED \ utf8proc_SOURCE=BUNDLED \ - ZSTD_SOURCE=BUNDLED + zstd_SOURCE=BUNDLED diff --git a/ci/docker/ubuntu-18.04-cpp.dockerfile b/ci/docker/ubuntu-18.04-cpp.dockerfile index a0fe1b3f6bec..bfff20b441c9 100644 --- a/ci/docker/ubuntu-18.04-cpp.dockerfile +++ b/ci/docker/ubuntu-18.04-cpp.dockerfile @@ -70,6 +70,7 @@ RUN apt-get update -y -q && \ libboost-system-dev \ libbrotli-dev \ libbz2-dev \ + libcurl4-openssl-dev \ libgflags-dev \ libgoogle-glog-dev \ liblz4-dev \ @@ -96,6 +97,7 @@ RUN apt-get update -y -q && \ # - libgtest-dev only provide sources # - libprotobuf-dev only provide sources # - thrift is too old +# - s3 tests would require boost-asio that is included since Boost 1.66.0 ENV ARROW_BUILD_TESTS=ON \ ARROW_DEPENDENCY_SOURCE=SYSTEM \ ARROW_DATASET=ON \ @@ -117,6 +119,7 @@ ENV ARROW_BUILD_TESTS=ON \ ARROW_WITH_SNAPPY=ON \ ARROW_WITH_ZLIB=ON \ ARROW_WITH_ZSTD=ON \ + AWSSDK_SOURCE=BUNDLED \ GTest_SOURCE=BUNDLED \ ORC_SOURCE=BUNDLED \ PARQUET_BUILD_EXECUTABLES=ON \ diff --git a/ci/docker/ubuntu-20.04-cpp.dockerfile b/ci/docker/ubuntu-20.04-cpp.dockerfile index ce738f5e5545..d2a3fab34b98 100644 --- a/ci/docker/ubuntu-20.04-cpp.dockerfile +++ b/ci/docker/ubuntu-20.04-cpp.dockerfile @@ -17,6 +17,7 @@ ARG base=amd64/ubuntu:20.04 FROM ${base} +ARG arch SHELL ["/bin/bash", "-o", "pipefail", "-c"] @@ -57,6 +58,7 @@ RUN apt-get update -y -q && \ libbrotli-dev \ libbz2-dev \ libgflags-dev \ + libcurl4-openssl-dev \ libgoogle-glog-dev \ liblz4-dev \ libprotobuf-dev \ @@ -72,10 +74,15 @@ RUN apt-get update -y -q && \ pkg-config \ protobuf-compiler \ rapidjson-dev \ - tzdata && \ + tzdata \ + wget && \ apt-get clean && \ rm -rf /var/lib/apt/lists* +COPY ci/scripts/install_minio.sh \ + /arrow/ci/scripts/ +RUN /arrow/ci/scripts/install_minio.sh ${arch} linux latest /usr/local + # Prioritize system packages and local installation # The following dependencies will be downloaded due to missing/invalid packages # provided by the distribution: @@ -95,6 +102,7 @@ ENV ARROW_BUILD_TESTS=ON \ ARROW_ORC=ON \ ARROW_PARQUET=ON \ ARROW_PLASMA=ON \ + ARROW_S3=ON \ ARROW_USE_ASAN=OFF \ ARROW_USE_CCACHE=ON \ ARROW_USE_UBSAN=OFF \ @@ -104,6 +112,7 @@ ENV ARROW_BUILD_TESTS=ON \ ARROW_WITH_SNAPPY=ON \ ARROW_WITH_ZLIB=ON \ ARROW_WITH_ZSTD=ON \ + AWSSDK_SOURCE=BUNDLED \ GTest_SOURCE=BUNDLED \ ORC_SOURCE=BUNDLED \ PARQUET_BUILD_EXAMPLES=ON \ diff --git a/ci/etc/integration_spark_ARROW-9438.patch b/ci/etc/integration_spark_ARROW-9438.patch deleted file mode 100644 index 2baed3037176..000000000000 --- a/ci/etc/integration_spark_ARROW-9438.patch +++ /dev/null @@ -1,72 +0,0 @@ -From 0b5388a945a7e5c5706cf00d0754540a6c68254d Mon Sep 17 00:00:00 2001 -From: Bryan Cutler -Date: Mon, 13 Jul 2020 23:12:25 -0700 -Subject: [PATCH] Update Arrow Java for 1.0.0 - ---- - pom.xml | 17 ++++++++++++++--- - sql/catalyst/pom.xml | 4 ++++ - 2 files changed, 18 insertions(+), 3 deletions(-) - -diff --git a/pom.xml b/pom.xml -index 08ca13bfe9..6619fca200 100644 ---- a/pom.xml -+++ b/pom.xml -@@ -199,7 +199,7 @@ - If you are changing Arrow version specification, please check ./python/pyspark/sql/utils.py, - and ./python/setup.py too. - --> -- 0.15.1 -+ 1.0.0-SNAPSHOT - - org.fusesource.leveldbjni - -@@ -2288,7 +2288,7 @@ - - - com.fasterxml.jackson.core -- jackson-databind -+ jackson-core - - - io.netty -@@ -2298,9 +2298,20 @@ - io.netty - netty-common - -+ -+ -+ -+ org.apache.arrow -+ arrow-memory-netty -+ ${arrow.version} -+ - - io.netty -- netty-handler -+ netty-buffer -+ -+ -+ io.netty -+ netty-common - - - -diff --git a/sql/catalyst/pom.xml b/sql/catalyst/pom.xml -index 9edbb7fec9..6b79eb722f 100644 ---- a/sql/catalyst/pom.xml -+++ b/sql/catalyst/pom.xml -@@ -117,6 +117,10 @@ - org.apache.arrow - arrow-vector - -+ -+ org.apache.arrow -+ arrow-memory-netty -+ - - - target/scala-${scala.binary.version}/classes --- -2.17.1 - diff --git a/ci/scripts/PKGBUILD b/ci/scripts/PKGBUILD index a813b2357f5d..2028b38dacab 100644 --- a/ci/scripts/PKGBUILD +++ b/ci/scripts/PKGBUILD @@ -18,13 +18,14 @@ _realname=arrow pkgbase=mingw-w64-${_realname} pkgname="${MINGW_PACKAGE_PREFIX}-${_realname}" -pkgver=1.0.0.9000 +pkgver=2.0.0 pkgrel=8000 pkgdesc="Apache Arrow is a cross-language development platform for in-memory data (mingw-w64)" arch=("any") url="https://arrow.apache.org/" license=("Apache-2.0") -depends=("${MINGW_PACKAGE_PREFIX}-thrift" +depends=("${MINGW_PACKAGE_PREFIX}-aws-sdk-cpp" + "${MINGW_PACKAGE_PREFIX}-thrift" "${MINGW_PACKAGE_PREFIX}-snappy" "${MINGW_PACKAGE_PREFIX}-zlib" "${MINGW_PACKAGE_PREFIX}-lz4" @@ -74,6 +75,9 @@ build() { export PATH="/C/Rtools${MINGW_PREFIX/mingw/mingw_}/bin:$PATH" export CPPFLAGS="${CPPFLAGS} -I${MINGW_PREFIX}/include" export LIBS="-L${MINGW_PREFIX}/libs" + export ARROW_S3=OFF + else + export ARROW_S3=ON fi MSYS2_ARG_CONV_EXCL="-DCMAKE_INSTALL_PREFIX=" \ @@ -90,15 +94,19 @@ build() { -DARROW_HDFS=OFF \ -DARROW_JEMALLOC=OFF \ -DARROW_JSON=ON \ + -DARROW_LZ4_USE_SHARED=OFF \ -DARROW_MIMALLOC=ON \ -DARROW_PACKAGE_PREFIX="${MINGW_PREFIX}" \ -DARROW_PARQUET=ON \ + -DARROW_S3="${ARROW_S3}" \ + -DARROW_SNAPPY_USE_SHARED=OFF \ -DARROW_USE_GLOG=OFF \ -DARROW_WITH_LZ4=ON \ -DARROW_WITH_SNAPPY=ON \ -DARROW_WITH_UTF8PROC=OFF \ -DARROW_WITH_ZLIB=ON \ -DARROW_WITH_ZSTD=ON \ + -DARROW_ZSTD_USE_SHARED=OFF \ -DARROW_CXXFLAGS="${CPPFLAGS}" \ -DCMAKE_BUILD_TYPE="release" \ -DCMAKE_INSTALL_PREFIX=${MINGW_PREFIX} \ diff --git a/ci/scripts/cpp_build.sh b/ci/scripts/cpp_build.sh index 2de6254aacc3..fe109b77b09e 100755 --- a/ci/scripts/cpp_build.sh +++ b/ci/scripts/cpp_build.sh @@ -82,6 +82,7 @@ cmake -G "${CMAKE_GENERATOR:-Ninja}" \ -DARROW_PLASMA_JAVA_CLIENT=${ARROW_PLASMA_JAVA_CLIENT:-OFF} \ -DARROW_PLASMA=${ARROW_PLASMA:-OFF} \ -DARROW_PYTHON=${ARROW_PYTHON:-OFF} \ + -DARROW_RUNTIME_SIMD_LEVEL=${ARROW_RUNTIME_SIMD_LEVEL:-MAX} \ -DARROW_S3=${ARROW_S3:-OFF} \ -DARROW_TEST_LINKAGE=${ARROW_TEST_LINKAGE:-shared} \ -DARROW_TEST_MEMCHECK=${ARROW_TEST_MEMCHECK:-OFF} \ @@ -100,6 +101,7 @@ cmake -G "${CMAKE_GENERATOR:-Ninja}" \ -DARROW_WITH_UTF8PROC=${ARROW_WITH_UTF8PROC:-ON} \ -DARROW_WITH_ZLIB=${ARROW_WITH_ZLIB:-OFF} \ -DARROW_WITH_ZSTD=${ARROW_WITH_ZSTD:-OFF} \ + -DAWSSDK_SOURCE=${AWSSDK_SOURCE:-} \ -Dbenchmark_SOURCE=${benchmark_SOURCE:-} \ -DBOOST_SOURCE=${BOOST_SOURCE:-} \ -DBrotli_SOURCE=${Brotli_SOURCE:-} \ @@ -125,7 +127,7 @@ cmake -G "${CMAKE_GENERATOR:-Ninja}" \ -DSnappy_SOURCE=${Snappy_SOURCE:-} \ -DThrift_SOURCE=${Thrift_SOURCE:-} \ -Dutf8proc_SOURCE=${utf8proc_SOURCE:-} \ - -DZSTD_SOURCE=${ZSTD_SOURCE:-} \ + -Dzstd_SOURCE=${zstd_SOURCE:-} \ ${CMAKE_ARGS} \ ${source_dir} diff --git a/ci/scripts/install_minio.sh b/ci/scripts/install_minio.sh index 9ed70afc03b8..42f7ce040e04 100755 --- a/ci/scripts/install_minio.sh +++ b/ci/scripts/install_minio.sh @@ -20,11 +20,14 @@ set -e declare -A archs -archs=([amd64]=amd64) +archs=([amd64]=amd64 + [arm64v8]=arm64 + [arm32v7]=arm + [s390x]=s390x) declare -A platforms -platforms=([macos]=darwin - [linux]=linux) +platforms=([linux]=linux + [macos]=darwin) arch=${archs[$1]} platform=${platforms[$2]} @@ -34,10 +37,10 @@ prefix=$4 if [ "$#" -ne 4 ]; then echo "Usage: $0 " exit 1 -elif [[ -z ${archs[$1]} ]]; then +elif [[ -z ${arch} ]]; then echo "Unexpected architecture: ${1}" exit 1 -elif [[ -z ${platforms[$2]} ]]; then +elif [[ -z ${platform} ]]; then echo "Unexpected platform: ${2}" exit 1 elif [[ ${version} != "latest" ]]; then @@ -45,5 +48,5 @@ elif [[ ${version} != "latest" ]]; then exit 1 fi -wget -nv -P ${prefix}/bin https://dl.min.io/server/minio/release/linux-${arch}/minio +wget -nv -P ${prefix}/bin https://dl.min.io/server/minio/release/${platform}-${arch}/minio chmod +x ${prefix}/bin/minio diff --git a/ci/scripts/integration_kartothek.sh b/ci/scripts/integration_kartothek.sh index f1465ba40e66..6e89f7263398 100755 --- a/ci/scripts/integration_kartothek.sh +++ b/ci/scripts/integration_kartothek.sh @@ -27,4 +27,4 @@ python -c "import pyarrow.parquet" python -c "import kartothek" pushd /kartothek -pytest -n0 +pytest -n0 --ignore tests/cli/test_query.py diff --git a/ci/scripts/integration_spark.sh b/ci/scripts/integration_spark.sh index 9828a28a1ec9..a45ed7a71254 100755 --- a/ci/scripts/integration_spark.sh +++ b/ci/scripts/integration_spark.sh @@ -22,6 +22,9 @@ source_dir=${1} spark_dir=${2} spark_version=${SPARK_VERSION:-master} +# Use old behavior that always dropped tiemzones. +export PYARROW_IGNORE_TIMEZONE=1 + if [ "${SPARK_VERSION:0:2}" == "2." ]; then # https://github.com/apache/spark/blob/master/docs/sql-pyspark-pandas-with-arrow.md#compatibility-setting-for-pyarrow--0150-and-spark-23x-24x export ARROW_PRE_0_15_IPC_FORMAT=1 diff --git a/ci/scripts/java_build.sh b/ci/scripts/java_build.sh index b5643f778402..a2deafa17ba4 100755 --- a/ci/scripts/java_build.sh +++ b/ci/scripts/java_build.sh @@ -23,6 +23,62 @@ source_dir=${1}/java cpp_build_dir=${2}/cpp/${ARROW_BUILD_TYPE:-debug} with_docs=${3:-false} +if [[ "$(uname -s)" == "Linux" ]] && [[ "$(uname -m)" == "s390x" ]]; then + # Since some files for s390_64 are not available at maven central, + # download pre-build files from bintray and install them explicitly + mvn_install="mvn install:install-file" + wget="wget" + bintray_base_url="https://dl.bintray.com/apache/arrow" + + bintray_dir="flatc-binary" + group="com.github.icexelloss" + artifact="flatc-linux-s390_64" + ver="1.9.0" + extension="exe" + target=${artifact}-${ver}.${extension} + ${wget} ${bintray_base_url}/${bintray_dir}/${ver}/${target} + ${mvn_install} -DgroupId=${group} -DartifactId=${artifact} -Dversion=${ver} -Dpackaging=${extension} -Dfile=$(pwd)/${target} + + bintray_dir="protoc-binary" + group="com.google.protobuf" + artifact="protoc" + ver="3.7.1" + classifier="linux-s390_64" + extension="exe" + target=${artifact}-${ver}-${classifier}.${extension} + ${wget} ${bintray_base_url}/${bintray_dir}/${ver}/${target} + ${mvn_install} -DgroupId=${group} -DartifactId=${artifact} -Dversion=${ver} -Dclassifier=${classifier} -Dpackaging=${extension} -Dfile=$(pwd)/${target} + # protoc requires libprotoc.so.18 libprotobuf.so.18 + ${wget} ${bintray_base_url}/${bintray_dir}/${ver}/libprotoc.so.18 + ${wget} ${bintray_base_url}/${bintray_dir}/${ver}/libprotobuf.so.18 + export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:$(pwd) + + bintray_dir="protoc-gen-grpc-java-binary" + group="io.grpc" + artifact="protoc-gen-grpc-java" + ver="1.30.2" + classifier="linux-s390_64" + extension="exe" + target=${artifact}-${ver}-${classifier}.${extension} + ${wget} ${bintray_base_url}/${bintray_dir}/${ver}/${target} + ${mvn_install} -DgroupId=${group} -DartifactId=${artifact} -Dversion=${ver} -Dclassifier=${classifier} -Dpackaging=${extension} -Dfile=$(pwd)/${target} + + bintray_dir="netty-binary" + group="io.netty" + artifact="netty-transport-native-unix-common" + ver="4.1.48.Final" + classifier="linux-s390_64" + extension="jar" + target=${artifact}-${ver}-${classifier}.${extension} + ${wget} ${bintray_base_url}/${bintray_dir}/${ver}/${target} + ${mvn_install} -DgroupId=${group} -DartifactId=${artifact} -Dversion=${ver} -Dclassifier=${classifier} -Dpackaging=${extension} -Dfile=$(pwd)/${target} + artifact="netty-transport-native-epoll" + extension="jar" + target=${artifact}-${ver}-${classifier}.${extension} + ${wget} ${bintray_base_url}/${bintray_dir}/${ver}/${target} + ${mvn_install} -DgroupId=${group} -DartifactId=${artifact} -Dversion=${ver} -Dclassifier=${classifier} -Dpackaging=${extension} -Dfile=$(pwd)/${target} +fi + mvn="mvn -B -DskipTests -Drat.skip=true -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn" # Use `2 * ncores` threads mvn="${mvn} -T 2C" diff --git a/ci/scripts/msys2_setup.sh b/ci/scripts/msys2_setup.sh index a2845d1473bd..110b03bae6cc 100755 --- a/ci/scripts/msys2_setup.sh +++ b/ci/scripts/msys2_setup.sh @@ -25,6 +25,7 @@ packages=() case "${target}" in cpp|c_glib|ruby) packages+=(make) + packages+=(${MINGW_PACKAGE_PREFIX}-aws-sdk-cpp) packages+=(${MINGW_PACKAGE_PREFIX}-ccache) packages+=(${MINGW_PACKAGE_PREFIX}-boost) packages+=(${MINGW_PACKAGE_PREFIX}-brotli) diff --git a/ci/scripts/msys2_system_upgrade_phase1.sh b/ci/scripts/msys2_system_upgrade_phase1.sh index 0839228f4196..aecd30893320 100755 --- a/ci/scripts/msys2_system_upgrade_phase1.sh +++ b/ci/scripts/msys2_system_upgrade_phase1.sh @@ -20,6 +20,9 @@ set -eux # https://www.msys2.org/news/#2020-06-29-new-packagers +msys2_repo_base_url=https://repo.msys2.org/msys +# Mirror +msys2_repo_base_url=https://sourceforge.net/projects/msys2/files/REPOS/MSYS2 msys2_keyring_pkg=msys2-keyring-r21.b39fb11-1-any.pkg.tar.xz for suffix in "" ".sig"; do curl \ @@ -27,7 +30,7 @@ for suffix in "" ".sig"; do --remote-name \ --show-error \ --silent \ - https://repo.msys2.org/msys/x86_64/${msys2_keyring_pkg}${suffix} + ${msys2_repo_base_url}/x86_64/${msys2_keyring_pkg}${suffix} done pacman-key --verify ${msys2_keyring_pkg}.sig pacman \ diff --git a/ci/scripts/python_test.sh b/ci/scripts/python_test.sh index 6f961d2f8e0f..80a9cdef4a31 100755 --- a/ci/scripts/python_test.sh +++ b/ci/scripts/python_test.sh @@ -29,4 +29,4 @@ export LD_LIBRARY_PATH=${ARROW_HOME}/lib:${LD_LIBRARY_PATH} # Enable some checks inside Python itself export PYTHONDEVMODE=1 -pytest -r s --pyargs pyarrow +pytest -r s ${PYTEST_ARGS} --pyargs pyarrow diff --git a/ci/scripts/r_deps.sh b/ci/scripts/r_deps.sh index a2dc58fd97bc..7e9d2eac7a96 100755 --- a/ci/scripts/r_deps.sh +++ b/ci/scripts/r_deps.sh @@ -25,7 +25,7 @@ source_dir=${1}/r pushd ${source_dir} # Install R package dependencies -${R_BIN} -e "install.packages('remotes'); remotes::install_cran(c('glue', 'rcmdcheck'))" +${R_BIN} -e "install.packages('remotes'); remotes::install_cran(c('glue', 'rcmdcheck', 'sys'))" ${R_BIN} -e "remotes::install_deps(dependencies = TRUE)" popd diff --git a/ci/scripts/r_docker_configure.sh b/ci/scripts/r_docker_configure.sh index 1d7e8de8bf5c..e6594e03a887 100755 --- a/ci/scripts/r_docker_configure.sh +++ b/ci/scripts/r_docker_configure.sh @@ -39,6 +39,25 @@ if [ "$RHUB_PLATFORM" = "linux-x86_64-fedora-clang" ]; then rm -rf $(${R_BIN} RHOME)/etc/Makeconf.bak fi +# Install openssl for S3 support +if [ "$ARROW_S3" == "ON" ] || [ "$ARROW_R_DEV" == "TRUE" ]; then + if [ "`which dnf`" ]; then + dnf install -y libcurl-devel openssl-devel + elif [ "`which yum`" ]; then + yum install -y libcurl-devel openssl-devel + elif [ "`which zypper`" ]; then + zypper install -y libcurl-devel libopenssl-devel + else + apt-get update + apt-get install -y libcurl4-openssl-dev libssl-dev + fi + + # The Dockerfile should have put this file here + if [ -f "/arrow/ci/scripts/install_minio.sh" ] && [ "`which wget`" ]; then + /arrow/ci/scripts/install_minio.sh amd64 linux latest /usr/local + fi +fi + # Workaround for html help install failure; see https://github.com/r-lib/devtools/issues/2084#issuecomment-530912786 Rscript -e 'x <- file.path(R.home("doc"), "html"); if (!file.exists(x)) {dir.create(x, recursive=TRUE); file.copy(system.file("html/R.css", package="stats"), x)}' diff --git a/ci/scripts/r_test.sh b/ci/scripts/r_test.sh index 05c70d8a560c..a2428e912bea 100755 --- a/ci/scripts/r_test.sh +++ b/ci/scripts/r_test.sh @@ -59,6 +59,13 @@ ${R_BIN} -e "as_cran <- !identical(tolower(Sys.getenv('NOT_CRAN')), 'true') if (as_cran) { rcmdcheck::rcmdcheck(args = c('--as-cran', '--run-donttest'), error_on = 'warning', check_dir = 'check') } else { + if (nzchar(Sys.which('minio'))) { + message('Running minio for S3 tests (if build supports them)') + minio_dir <- tempfile() + dir.create(minio_dir) + pid <- sys::exec_background('minio', c('server', minio_dir)) + on.exit(tools::pskill(pid)) + } rcmdcheck::rcmdcheck(build_args = '--no-build-vignettes', args = c('--no-manual', '--ignore-vignettes', '--run-donttest'), error_on = 'warning', check_dir = 'check') }" diff --git a/ci/scripts/r_windows_build.sh b/ci/scripts/r_windows_build.sh index ed9e211f9acb..cb33e676a7dc 100755 --- a/ci/scripts/r_windows_build.sh +++ b/ci/scripts/r_windows_build.sh @@ -27,13 +27,25 @@ if [ "$RTOOLS_VERSION" = "35" ]; then # Use rtools-backports if building with rtools35 curl https://raw.githubusercontent.com/r-windows/rtools-backports/master/pacman.conf > /etc/pacman.conf # Update keys: https://www.msys2.org/news/#2020-06-29-new-packagers - curl -OSsl "http://repo.msys2.org/msys/x86_64/msys2-keyring-r21.b39fb11-1-any.pkg.tar.xz" + msys2_repo_base_url=https://repo.msys2.org/msys + # Mirror + msys2_repo_base_url=https://sourceforge.net/projects/msys2/files/REPOS/MSYS2 + curl -OSsL "${msys2_repo_base_url}/x86_64/msys2-keyring-r21.b39fb11-1-any.pkg.tar.xz" pacman -U --noconfirm msys2-keyring-r21.b39fb11-1-any.pkg.tar.xz && rm msys2-keyring-r21.b39fb11-1-any.pkg.tar.xz + # Use sf.net instead of http://repo.msys2.org/ temporary. + sed -i -e "s,^Server = http://repo\.msys2\.org/msys,Server = ${msys2_repo_base_url},g" \ + /etc/pacman.conf pacman --noconfirm -Scc pacman --noconfirm -Syy # lib-4.9.3 is for libraries compiled with gcc 4.9 (Rtools 3.5) RWINLIB_LIB_DIR="lib-4.9.3" else + # Uncomment L38-41 if you're testing a new rtools dependency that hasn't yet sync'd to CRAN + # curl https://raw.githubusercontent.com/r-windows/rtools-packages/master/pacman.conf > /etc/pacman.conf + # curl -OSsl "http://repo.msys2.org/msys/x86_64/msys2-keyring-r21.b39fb11-1-any.pkg.tar.xz" + # pacman -U --noconfirm msys2-keyring-r21.b39fb11-1-any.pkg.tar.xz && rm msys2-keyring-r21.b39fb11-1-any.pkg.tar.xz + # pacman --noconfirm -Scc + pacman --noconfirm -Syy RWINLIB_LIB_DIR="lib" fi @@ -84,8 +96,8 @@ cp $MSYS_LIB_DIR/mingw64/lib/lib{thrift,snappy}.a $DST_DIR/${RWINLIB_LIB_DIR}/x6 cp $MSYS_LIB_DIR/mingw32/lib/lib{thrift,snappy}.a $DST_DIR/${RWINLIB_LIB_DIR}/i386 # These are from https://dl.bintray.com/rtools/mingw{32,64}/ -cp $MSYS_LIB_DIR/mingw64/lib/lib{zstd,lz4,crypto}.a $DST_DIR/lib/x64 -cp $MSYS_LIB_DIR/mingw32/lib/lib{zstd,lz4,crypto}.a $DST_DIR/lib/i386 +cp $MSYS_LIB_DIR/mingw64/lib/lib{zstd,lz4,crypto,aws*}.a $DST_DIR/lib/x64 +cp $MSYS_LIB_DIR/mingw32/lib/lib{zstd,lz4,crypto,aws*}.a $DST_DIR/lib/i386 # Create build artifact zip -r ${DST_DIR}.zip $DST_DIR diff --git a/dev/tasks/linux-packages/apache-arrow-archive-keyring/apt/ubuntu-eoan/Dockerfile b/ci/scripts/rust_coverage.sh old mode 100644 new mode 100755 similarity index 61% rename from dev/tasks/linux-packages/apache-arrow-archive-keyring/apt/ubuntu-eoan/Dockerfile rename to ci/scripts/rust_coverage.sh index 884710d61a0f..fbe5b0d853a5 --- a/dev/tasks/linux-packages/apache-arrow-archive-keyring/apt/ubuntu-eoan/Dockerfile +++ b/ci/scripts/rust_coverage.sh @@ -1,3 +1,5 @@ +#!/usr/bin/env bash +# # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information @@ -15,27 +17,23 @@ # specific language governing permissions and limitations # under the License. -FROM ubuntu:eoan +set -ex + +arrow_dir=${1} +source_dir=${1}/rust +build_dir=${2}/rust +rust=${3} + +export ARROW_TEST_DATA=${arrow_dir}/testing/data +export PARQUET_TEST_DATA=${arrow_dir}/cpp/submodules/parquet-testing/data +export CARGO_TARGET_DIR=${build_dir} -RUN \ - echo "debconf debconf/frontend select Noninteractive" | \ - debconf-set-selections +pushd ${source_dir} -RUN \ - echo 'APT::Install-Recommends "false";' > \ - /etc/apt/apt.conf.d/disable-install-recommends +rustup default ${rust} +rustup component add rustfmt --toolchain ${rust}-x86_64-unknown-linux-gnu +cargo install cargo-tarpaulin -ARG DEBUG +cargo tarpaulin --out Xml -RUN \ - quiet=$([ "${DEBUG}" = "yes" ] || echo "-qq") && \ - apt update ${quiet} && \ - apt install -y -V ${quiet} \ - build-essential \ - debhelper \ - devscripts \ - fakeroot \ - gnupg \ - lsb-release && \ - apt clean && \ - rm -rf /var/lib/apt/lists/* +popd diff --git a/cpp/Brewfile b/cpp/Brewfile index 6887956f273a..7de6c7deabeb 100644 --- a/cpp/Brewfile +++ b/cpp/Brewfile @@ -28,13 +28,16 @@ brew "grpc" brew "llvm" brew "llvm@8" brew "lz4" +brew "minio" brew "ninja" brew "numpy" brew "openssl@1.1" brew "protobuf" brew "python" brew "rapidjson" -brew "re2" +# grpc bundles re2 and causes a conflict when Homebrew tries to install it, +# so temporarily skip installing re2. See ARROW-9972. +# brew "re2" brew "snappy" brew "thrift" brew "wget" diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 55d5ed9c258f..515e6aff09d9 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -54,7 +54,7 @@ if(POLICY CMP0063) cmake_policy(SET CMP0063 NEW) endif() -set(ARROW_VERSION "2.0.0-SNAPSHOT") +set(ARROW_VERSION "2.0.0") string(REGEX MATCH "^[0-9]+\\.[0-9]+\\.[0-9]+" ARROW_BASE_VERSION "${ARROW_VERSION}") @@ -100,7 +100,7 @@ set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_CURRENT_SOURCE_DIR}/cmake_mo include(CMakePackageConfigHelpers) include(CMakeParseArguments) include(ExternalProject) -include(FindPkgConfig) +include(FindPackageHandleStandardArgs) include(GNUInstallDirs) @@ -148,6 +148,13 @@ if(APPLE) endif() endif() +if(WIN32 AND NOT MINGW) + # This is used to handle builds using e.g. clang in an MSVC setting. + set(MSVC_TOOLCHAIN TRUE) +else() + set(MSVC_TOOLCHAIN FALSE) +endif() + find_package(ClangTools) find_package(InferTools) if("$ENV{CMAKE_EXPORT_COMPILE_COMMANDS}" STREQUAL "1" OR CLANG_TIDY_FOUND OR INFER_FOUND) @@ -315,6 +322,7 @@ if(ARROW_BUILD_BENCHMARKS OR ARROW_BUILD_INTEGRATION OR ARROW_FUZZING) set(ARROW_JSON ON) + set(ARROW_TESTING ON) endif() if(ARROW_CUDA @@ -343,7 +351,7 @@ if(ARROW_PYTHON) set(ARROW_JSON ON) endif() -if(MSVC) +if(MSVC_TOOLCHAIN) # ORC doesn't build on windows set(ARROW_ORC OFF) # Plasma using glog is not fully tested on windows. @@ -410,6 +418,10 @@ if(ARROW_TEST_MEMCHECK) add_definitions(-DARROW_VALGRIND) endif() +if(ARROW_USE_UBSAN) + add_definitions(-DARROW_UBSAN) +endif() + # # Compiler flags # @@ -650,6 +662,7 @@ if(ARROW_USE_OPENSSL) set(ARROW_OPENSSL_LIBS OpenSSL::Crypto OpenSSL::SSL) list(APPEND ARROW_LINK_LIBS ${ARROW_OPENSSL_LIBS}) list(APPEND ARROW_STATIC_LINK_LIBS ${ARROW_OPENSSL_LIBS}) + list(APPEND ARROW_STATIC_INSTALL_INTERFACE_LIBS ${ARROW_OPENSSL_LIBS}) endif() if(ARROW_WITH_BROTLI) @@ -657,46 +670,75 @@ if(ARROW_WITH_BROTLI) set(ARROW_BROTLI_LIBS Brotli::brotlienc Brotli::brotlidec Brotli::brotlicommon) list(APPEND ARROW_LINK_LIBS ${ARROW_BROTLI_LIBS}) list(APPEND ARROW_STATIC_LINK_LIBS ${ARROW_BROTLI_LIBS}) + if(Brotli_SOURCE STREQUAL "SYSTEM") + list(APPEND ARROW_STATIC_INSTALL_INTERFACE_LIBS ${ARROW_BROTLI_LIBS}) + endif() endif() if(ARROW_WITH_BZ2) list(APPEND ARROW_STATIC_LINK_LIBS BZip2::BZip2) + if(BZip2_SOURCE STREQUAL "SYSTEM") + list(APPEND ARROW_STATIC_INSTALL_INTERFACE_LIBS BZip2::BZip2) + endif() endif() if(ARROW_WITH_LZ4) list(APPEND ARROW_STATIC_LINK_LIBS LZ4::lz4) + if(Lz4_SOURCE STREQUAL "SYSTEM") + list(APPEND ARROW_STATIC_INSTALL_INTERFACE_LIBS LZ4::lz4) + endif() endif() if(ARROW_WITH_SNAPPY) list(APPEND ARROW_STATIC_LINK_LIBS Snappy::snappy) + if(Snappy_SOURCE STREQUAL "SYSTEM") + list(APPEND ARROW_STATIC_INSTALL_INTERFACE_LIBS Snappy::snappy) + endif() endif() if(ARROW_WITH_ZLIB) list(APPEND ARROW_STATIC_LINK_LIBS ZLIB::ZLIB) + if(ZLIB_SOURCE STREQUAL "SYSTEM") + list(APPEND ARROW_STATIC_INSTALL_INTERFACE_LIBS ZLIB::ZLIB) + endif() endif() if(ARROW_WITH_ZSTD) list(APPEND ARROW_STATIC_LINK_LIBS ${ARROW_ZSTD_LIBZSTD}) + if(zstd_SOURCE STREQUAL "SYSTEM") + list(APPEND ARROW_STATIC_INSTALL_INTERFACE_LIBS ${ARROW_ZSTD_LIBZSTD}) + endif() endif() if(ARROW_ORC) list(APPEND ARROW_LINK_LIBS orc::liborc ${ARROW_PROTOBUF_LIBPROTOBUF}) list(APPEND ARROW_STATIC_LINK_LIBS orc::liborc ${ARROW_PROTOBUF_LIBPROTOBUF}) + if(ORC_SOURCE STREQUAL "SYSTEM") + list(APPEND ARROW_STATIC_INSTALL_INTERFACE_LIBS orc::liborc + ${ARROW_PROTOBUF_LIBPROTOBUF}) + endif() endif() if(ARROW_USE_GLOG) list(APPEND ARROW_LINK_LIBS glog::glog) list(APPEND ARROW_STATIC_LINK_LIBS glog::glog) + if(GLOG_SOURCE STREQUAL "SYSTEM") + list(APPEND ARROW_STATIC_INSTALL_INTERFACE_LIBS glog::glog) + endif() add_definitions("-DARROW_USE_GLOG") endif() if(ARROW_S3) list(APPEND ARROW_LINK_LIBS ${AWSSDK_LINK_LIBRARIES}) + list(APPEND ARROW_STATIC_LINK_LIBS ${AWSSDK_LINK_LIBRARIES}) endif() if(ARROW_WITH_UTF8PROC) list(APPEND ARROW_LINK_LIBS utf8proc::utf8proc) list(APPEND ARROW_STATIC_LINK_LIBS utf8proc::utf8proc) + if(utf8proc_SOURCE STREQUAL "SYSTEM") + list(APPEND ARROW_STATIC_INSTALL_INTERFACE_LIBS utf8proc::utf8proc) + endif() endif() add_custom_target(arrow_dependencies) @@ -712,7 +754,7 @@ add_dependencies(arrow_test_dependencies toolchain-tests) if(ARROW_STATIC_LINK_LIBS) add_dependencies(arrow_dependencies ${ARROW_STATIC_LINK_LIBS}) if(ARROW_ORC) - if(NOT MSVC) + if(NOT MSVC_TOOLCHAIN) list(APPEND ARROW_STATIC_LINK_LIBS ${CMAKE_DL_LIBS}) list(APPEND ARROW_STATIC_INSTALL_INTERFACE_LIBS ${CMAKE_DL_LIBS}) endif() @@ -726,7 +768,7 @@ if(((ARROW_FLIGHT OR ARROW_S3) AND (ARROW_BUILD_TESTS OR ARROW_BUILD_INTEGRATION list(APPEND ARROW_TEST_LINK_LIBS ${BOOST_FILESYSTEM_LIBRARY} ${BOOST_SYSTEM_LIBRARY}) endif() -if(NOT MSVC) +if(NOT MSVC_TOOLCHAIN) list(APPEND ARROW_LINK_LIBS ${CMAKE_DL_LIBS}) list(APPEND ARROW_SHARED_INSTALL_INTERFACE_LIBS ${CMAKE_DL_LIBS}) endif() @@ -825,6 +867,7 @@ endif() list(APPEND ARROW_LINK_LIBS ${ARROW_SYSTEM_LINK_LIBS}) list(APPEND ARROW_STATIC_LINK_LIBS ${ARROW_SYSTEM_LINK_LIBS}) +list(APPEND ARROW_STATIC_INSTALL_INTERFACE_LIBS ${ARROW_SYSTEM_LINK_LIBS}) # # Subdirectories diff --git a/cpp/build-support/run-test.sh b/cpp/build-support/run-test.sh index 7e2a22f069a0..1cda600d1548 100755 --- a/cpp/build-support/run-test.sh +++ b/cpp/build-support/run-test.sh @@ -96,12 +96,14 @@ function run_test() { # even when retries are successful. rm -f $XMLFILE - $TEST_EXECUTABLE "$@" 2>&1 \ + $TEST_EXECUTABLE "$@" > $LOGFILE.raw 2>&1 + STATUS=$? + cat $LOGFILE.raw \ | ${PYTHON:-python} $ROOT/build-support/asan_symbolize.py \ | ${CXXFILT:-c++filt} \ | $ROOT/build-support/stacktrace_addr2line.pl $TEST_EXECUTABLE \ | $pipe_cmd 2>&1 | tee $LOGFILE - STATUS=$? + rm -f $LOGFILE.raw # TSAN doesn't always exit with a non-zero exit code due to a bug: # mutex errors don't get reported through the normal error reporting infrastructure. diff --git a/cpp/cmake_modules/BuildUtils.cmake b/cpp/cmake_modules/BuildUtils.cmake index eb10ebeeb1c7..e59b4a38af09 100644 --- a/cpp/cmake_modules/BuildUtils.cmake +++ b/cpp/cmake_modules/BuildUtils.cmake @@ -18,7 +18,7 @@ # Common path suffixes to be searched by find_library or find_path. # Windows artifacts may be found under "/Library", so # search there as well. -set(LIB_PATH_SUFFIXES +set(ARROW_LIBRARY_PATH_SUFFIXES "${CMAKE_LIBRARY_ARCHITECTURE}" "lib/${CMAKE_LIBRARY_ARCHITECTURE}" "lib64" @@ -28,7 +28,23 @@ set(LIB_PATH_SUFFIXES "Library" "Library/lib" "Library/bin") -set(INCLUDE_PATH_SUFFIXES "include" "Library" "Library/include") +set(ARROW_INCLUDE_PATH_SUFFIXES "include" "Library" "Library/include") + +set(ARROW_BOOST_PROCESS_COMPILE_DEFINITIONS) +if(WIN32 AND CMAKE_CXX_COMPILER_ID STREQUAL "GNU") + # boost/process/detail/windows/handle_workaround.hpp doesn't work + # without BOOST_USE_WINDOWS_H with MinGW because MinGW doesn't + # provide __kernel_entry without winternl.h. + # + # See also: + # https://github.com/boostorg/process/blob/develop/include/boost/process/detail/windows/handle_workaround.hpp + # + # You can use this like the following: + # + # target_compile_definitions(target PRIVATE + # ${ARROW_BOOST_PROCESS_COMPILE_DEFINITIONS}) + list(APPEND ARROW_BOOST_PROCESS_COMPILE_DEFINITIONS "BOOST_USE_WINDOWS_H=1") +endif() function(ADD_THIRDPARTY_LIB LIB_NAME) set(options) @@ -213,9 +229,14 @@ endfunction() # \arg OUTPUTS list to append built targets to function(ADD_ARROW_LIB LIB_NAME) - set(options BUILD_SHARED BUILD_STATIC) - set(one_value_args CMAKE_PACKAGE_NAME PKG_CONFIG_NAME SHARED_LINK_FLAGS - PRECOMPILED_HEADER_LIB) + set(options) + set(one_value_args + BUILD_SHARED + BUILD_STATIC + CMAKE_PACKAGE_NAME + PKG_CONFIG_NAME + SHARED_LINK_FLAGS + PRECOMPILED_HEADER_LIB) set(multi_value_args SOURCES PRECOMPILED_HEADERS @@ -243,12 +264,12 @@ function(ADD_ARROW_LIB LIB_NAME) endif() # Allow overriding ARROW_BUILD_SHARED and ARROW_BUILD_STATIC - if(ARG_BUILD_SHARED) + if(DEFINED ARG_BUILD_SHARED) set(BUILD_SHARED ${ARG_BUILD_SHARED}) else() set(BUILD_SHARED ${ARROW_BUILD_SHARED}) endif() - if(ARG_BUILD_STATIC) + if(DEFINED ARG_BUILD_STATIC) set(BUILD_STATIC ${ARG_BUILD_STATIC}) else() set(BUILD_STATIC ${ARROW_BUILD_STATIC}) @@ -334,7 +355,7 @@ function(ADD_ARROW_LIB LIB_NAME) endif() # On iOS, specifying -undefined conflicts with enabling bitcode - if(APPLE AND NOT IOS AND NOT DEFINED $ENV{EMSCRIPTEN}) + if(APPLE AND NOT IOS AND NOT DEFINED ENV{EMSCRIPTEN}) # On OS X, you can avoid linking at library load time and instead # expecting that the symbols have been loaded separately. This happens # with libpython* where there can be conflicts between system Python and @@ -419,7 +440,7 @@ function(ADD_ARROW_LIB LIB_NAME) target_include_directories(${LIB_NAME}_static PRIVATE ${ARG_PRIVATE_INCLUDES}) endif() - if(MSVC) + if(MSVC_TOOLCHAIN) set(LIB_NAME_STATIC ${LIB_NAME}_static) else() set(LIB_NAME_STATIC ${LIB_NAME}) diff --git a/cpp/cmake_modules/DefineOptions.cmake b/cpp/cmake_modules/DefineOptions.cmake index da2c6d551915..a68c3a92cc7f 100644 --- a/cpp/cmake_modules/DefineOptions.cmake +++ b/cpp/cmake_modules/DefineOptions.cmake @@ -105,15 +105,27 @@ if("${CMAKE_SOURCE_DIR}" STREQUAL "${CMAKE_CURRENT_SOURCE_DIR}") OFF) define_option_string(ARROW_SIMD_LEVEL - "SIMD compiler optimization level" + "Compile-time SIMD optimization level" "SSE4_2" # default to SSE4.2 "NONE" "SSE4_2" "AVX2" "AVX512") + define_option_string(ARROW_RUNTIME_SIMD_LEVEL + "Max runtime SIMD optimization level" + "MAX" # default to max supported by compiler + "NONE" + "SSE4_2" + "AVX2" + "AVX512" + "MAX") + # Arm64 architectures and extensions can lead to exploding combinations. # So set it directly through cmake command line. + # + # If you change this, you need to change the definition in + # python/CMakeLists.txt too. define_option_string(ARROW_ARMV8_ARCH "Arm64 arch and extensions" "armv8-a" # Default @@ -244,6 +256,8 @@ if("${CMAKE_SOURCE_DIR}" STREQUAL "${CMAKE_CURRENT_SOURCE_DIR}") define_option(ARROW_TENSORFLOW "Build Arrow with TensorFlow support enabled" OFF) + define_option(ARROW_TESTING "Build the Arrow testing libraries" OFF) + #---------------------------------------------------------------------- set_option_category("Thirdparty toolchain") @@ -284,29 +298,62 @@ if("${CMAKE_SOURCE_DIR}" STREQUAL "${CMAKE_CURRENT_SOURCE_DIR}") define_option(ARROW_VERBOSE_THIRDPARTY_BUILD "Show output from ExternalProjects rather than just logging to files" OFF) - define_option(ARROW_BOOST_USE_SHARED "Rely on boost shared libraries where relevant" ON) + define_option(ARROW_DEPENDENCY_USE_SHARED "Link to shared libraries" ON) + + define_option(ARROW_BOOST_USE_SHARED "Rely on boost shared libraries where relevant" + ${ARROW_DEPENDENCY_USE_SHARED}) define_option(ARROW_BROTLI_USE_SHARED "Rely on Brotli shared libraries where relevant" - ON) + ${ARROW_DEPENDENCY_USE_SHARED}) + + define_option(ARROW_BZ2_USE_SHARED "Rely on Bz2 shared libraries where relevant" + ${ARROW_DEPENDENCY_USE_SHARED}) define_option(ARROW_GFLAGS_USE_SHARED "Rely on GFlags shared libraries where relevant" - ON) + ${ARROW_DEPENDENCY_USE_SHARED}) + + define_option(ARROW_GRPC_USE_SHARED "Rely on gRPC shared libraries where relevant" + ${ARROW_DEPENDENCY_USE_SHARED}) + + define_option(ARROW_LZ4_USE_SHARED "Rely on lz4 shared libraries where relevant" + ${ARROW_DEPENDENCY_USE_SHARED}) - define_option(ARROW_GRPC_USE_SHARED "Rely on gRPC shared libraries where relevant" ON) + define_option(ARROW_OPENSSL_USE_SHARED "Rely on OpenSSL shared libraries where relevant" + ${ARROW_DEPENDENCY_USE_SHARED}) define_option(ARROW_PROTOBUF_USE_SHARED - "Rely on Protocol Buffers shared libraries where relevant" ON) + "Rely on Protocol Buffers shared libraries where relevant" + ${ARROW_DEPENDENCY_USE_SHARED}) + + if(WIN32) + # It seems that Thrift doesn't support DLL well yet. + # MSYS2, conda-forge and vcpkg don't build shared library. + set(ARROW_THRIFT_USE_SHARED_DEFAULT OFF) + else() + set(ARROW_THRIFT_USE_SHARED_DEFAULT ${ARROW_DEPENDENCY_USE_SHARED}) + endif() + define_option(ARROW_THRIFT_USE_SHARED "Rely on thrift shared libraries where relevant" + ${ARROW_THRIFT_USE_SHARED_DEFAULT}) define_option(ARROW_UTF8PROC_USE_SHARED - "Rely on utf8proc shared libraries where relevant" ON) + "Rely on utf8proc shared libraries where relevant" + ${ARROW_DEPENDENCY_USE_SHARED}) - define_option(ARROW_ZSTD_USE_SHARED "Rely on zstd shared libraries where relevant" ON) + define_option(ARROW_SNAPPY_USE_SHARED "Rely on snappy shared libraries where relevant" + ${ARROW_DEPENDENCY_USE_SHARED}) - define_option(ARROW_WITH_BACKTRACE "Build with backtrace support" ON) + define_option(ARROW_UTF8PROC_USE_SHARED + "Rely on utf8proc shared libraries where relevant" + ${ARROW_DEPENDENCY_USE_SHARED}) + + define_option(ARROW_ZSTD_USE_SHARED "Rely on zstd shared libraries where relevant" + ${ARROW_DEPENDENCY_USE_SHARED}) define_option(ARROW_USE_GLOG "Build libraries with glog support for pluggable logging" OFF) + define_option(ARROW_WITH_BACKTRACE "Build with backtrace support" ON) + define_option(ARROW_WITH_BROTLI "Build with Brotli compression" OFF) define_option(ARROW_WITH_BZ2 "Build with BZ2 compression" OFF) define_option(ARROW_WITH_LZ4 "Build with lz4 compression" OFF) @@ -318,7 +365,7 @@ if("${CMAKE_SOURCE_DIR}" STREQUAL "${CMAKE_CURRENT_SOURCE_DIR}") "Build with support for Unicode properties using the utf8proc library" ON) #---------------------------------------------------------------------- - if(MSVC) + if(MSVC_TOOLCHAIN) set_option_category("MSVC") define_option(MSVC_LINK_VERBOSE @@ -334,8 +381,16 @@ if("${CMAKE_SOURCE_DIR}" STREQUAL "${CMAKE_CURRENT_SOURCE_DIR}") define_option_string(RE2_MSVC_STATIC_LIB_SUFFIX "re2 static lib suffix used on Windows with MSVC" "_static") + if(DEFINED ENV{CONDA_PREFIX}) + # Conda package changes the output name. + # https://github.com/conda-forge/snappy-feedstock/blob/master/recipe/windows-static-lib-name.patch + set(SNAPPY_MSVC_STATIC_LIB_SUFFIX_DEFAULT "_static") + else() + set(SNAPPY_MSVC_STATIC_LIB_SUFFIX_DEFAULT "") + endif() define_option_string(SNAPPY_MSVC_STATIC_LIB_SUFFIX - "Snappy static lib suffix used on Windows with MSVC" "_static") + "Snappy static lib suffix used on Windows with MSVC" + "${SNAPPY_MSVC_STATIC_LIB_SUFFIX_DEFAULT}") define_option_string(LZ4_MSVC_STATIC_LIB_SUFFIX "Lz4 static lib suffix used on Windows with MSVC" "_static") diff --git a/cpp/cmake_modules/FindArrow.cmake b/cpp/cmake_modules/FindArrow.cmake index 02fd9a158010..9c987665896d 100644 --- a/cpp/cmake_modules/FindArrow.cmake +++ b/cpp/cmake_modules/FindArrow.cmake @@ -39,6 +39,13 @@ endif() include(FindPkgConfig) include(FindPackageHandleStandardArgs) +if(WIN32 AND NOT MINGW) + # This is used to handle builds using e.g. clang in an MSVC setting. + set(MSVC_TOOLCHAIN TRUE) +else() + set(MSVC_TOOLCHAIN FALSE) +endif() + set(ARROW_SEARCH_LIB_PATH_SUFFIXES) if(CMAKE_LIBRARY_ARCHITECTURE) list(APPEND ARROW_SEARCH_LIB_PATH_SUFFIXES "lib/${CMAKE_LIBRARY_ARCHITECTURE}") @@ -61,7 +68,7 @@ if(CMAKE_BUILD_TYPE) endif() if(NOT DEFINED ARROW_MSVC_STATIC_LIB_SUFFIX) - if(MSVC) + if(MSVC_TOOLCHAIN) set(ARROW_MSVC_STATIC_LIB_SUFFIX "_static") else() set(ARROW_MSVC_STATIC_LIB_SUFFIX "") @@ -147,7 +154,7 @@ macro(arrow_find_package_home) set(include_dir "${${prefix}_include_dir}") set(${prefix}_INCLUDE_DIR "${include_dir}" PARENT_SCOPE) - if(MSVC) + if(MSVC_TOOLCHAIN) set(CMAKE_SHARED_LIBRARY_SUFFIXES_ORIGINAL ${CMAKE_FIND_LIBRARY_SUFFIXES}) # .dll isn't found by find_library with MSVC because .dll isn't included in # CMAKE_FIND_LIBRARY_SUFFIXES. @@ -158,7 +165,7 @@ macro(arrow_find_package_home) PATHS "${home}" PATH_SUFFIXES ${ARROW_SEARCH_LIB_PATH_SUFFIXES} NO_DEFAULT_PATH) - if(MSVC) + if(MSVC_TOOLCHAIN) set(CMAKE_SHARED_LIBRARY_SUFFIXES ${CMAKE_FIND_LIBRARY_SUFFIXES_ORIGINAL}) endif() set(shared_lib "${${prefix}_shared_lib}") diff --git a/cpp/cmake_modules/FindBoostAlt.cmake b/cpp/cmake_modules/FindBoostAlt.cmake index 8f16439bf865..300080d4fb1b 100644 --- a/cpp/cmake_modules/FindBoostAlt.cmake +++ b/cpp/cmake_modules/FindBoostAlt.cmake @@ -52,7 +52,7 @@ endif() if(Boost_FOUND) set(BoostAlt_FOUND ON) - if(MSVC) + if(MSVC_TOOLCHAIN) # disable autolinking in boost add_definitions(-DBOOST_ALL_NO_LIB) if(ARROW_BOOST_USE_SHARED) diff --git a/cpp/cmake_modules/FindBrotli.cmake b/cpp/cmake_modules/FindBrotli.cmake index dc74c5ce58b2..b46a0f1a0cf4 100644 --- a/cpp/cmake_modules/FindBrotli.cmake +++ b/cpp/cmake_modules/FindBrotli.cmake @@ -51,24 +51,25 @@ if(BROTLI_ROOT) find_library(BROTLI_COMMON_LIBRARY NAMES ${BROTLI_COMMON_LIB_NAMES} PATHS ${BROTLI_ROOT} - PATH_SUFFIXES ${LIB_PATH_SUFFIXES} + PATH_SUFFIXES ${ARROW_LIBRARY_PATH_SUFFIXES} NO_DEFAULT_PATH) find_library(BROTLI_ENC_LIBRARY NAMES ${BROTLI_ENC_LIB_NAMES} PATHS ${BROTLI_ROOT} - PATH_SUFFIXES ${LIB_PATH_SUFFIXES} + PATH_SUFFIXES ${ARROW_LIBRARY_PATH_SUFFIXES} NO_DEFAULT_PATH) find_library(BROTLI_DEC_LIBRARY NAMES ${BROTLI_DEC_LIB_NAMES} PATHS ${BROTLI_ROOT} - PATH_SUFFIXES ${LIB_PATH_SUFFIXES} + PATH_SUFFIXES ${ARROW_LIBRARY_PATH_SUFFIXES} NO_DEFAULT_PATH) find_path(BROTLI_INCLUDE_DIR NAMES brotli/decode.h PATHS ${BROTLI_ROOT} - PATH_SUFFIXES ${INCLUDE_PATH_SUFFIXES} + PATH_SUFFIXES ${ARROW_INCLUDE_PATH_SUFFIXES} NO_DEFAULT_PATH) else() + find_package(PkgConfig QUIET) pkg_check_modules(BROTLI_PC libbrotlicommon libbrotlienc libbrotlidec) if(BROTLI_PC_FOUND) set(BROTLI_INCLUDE_DIR "${BROTLI_PC_libbrotlicommon_INCLUDEDIR}") @@ -81,31 +82,31 @@ else() find_library(BROTLI_COMMON_LIBRARY NAMES ${BROTLI_COMMON_LIB_NAMES} PATHS ${BROTLI_PC_LIBRARY_DIRS} - PATH_SUFFIXES ${LIB_PATH_SUFFIXES} + PATH_SUFFIXES ${ARROW_LIBRARY_PATH_SUFFIXES} NO_DEFAULT_PATH) find_library(BROTLI_ENC_LIBRARY NAMES ${BROTLI_ENC_LIB_NAMES} PATHS ${BROTLI_PC_LIBRARY_DIRS} - PATH_SUFFIXES ${LIB_PATH_SUFFIXES} + PATH_SUFFIXES ${ARROW_LIBRARY_PATH_SUFFIXES} NO_DEFAULT_PATH) find_library(BROTLI_DEC_LIBRARY NAMES ${BROTLI_DEC_LIB_NAMES} PATHS ${BROTLI_PC_LIBRARY_DIRS} - PATH_SUFFIXES ${LIB_PATH_SUFFIXES} + PATH_SUFFIXES ${ARROW_LIBRARY_PATH_SUFFIXES} NO_DEFAULT_PATH) else() find_library(BROTLI_COMMON_LIBRARY NAMES ${BROTLI_COMMON_LIB_NAMES} - PATH_SUFFIXES ${LIB_PATH_SUFFIXES}) + PATH_SUFFIXES ${ARROW_LIBRARY_PATH_SUFFIXES}) find_library(BROTLI_ENC_LIBRARY NAMES ${BROTLI_ENC_LIB_NAMES} - PATH_SUFFIXES ${LIB_PATH_SUFFIXES}) + PATH_SUFFIXES ${ARROW_LIBRARY_PATH_SUFFIXES}) find_library(BROTLI_DEC_LIBRARY NAMES ${BROTLI_DEC_LIB_NAMES} - PATH_SUFFIXES ${LIB_PATH_SUFFIXES}) + PATH_SUFFIXES ${ARROW_LIBRARY_PATH_SUFFIXES}) find_path(BROTLI_INCLUDE_DIR NAMES brotli/decode.h - PATH_SUFFIXES ${INCLUDE_PATH_SUFFIXES}) + PATH_SUFFIXES ${ARROW_INCLUDE_PATH_SUFFIXES}) endif() endif() diff --git a/cpp/cmake_modules/FindGLOG.cmake b/cpp/cmake_modules/FindGLOG.cmake index e64a22f1f385..81c3f2ec57e2 100644 --- a/cpp/cmake_modules/FindGLOG.cmake +++ b/cpp/cmake_modules/FindGLOG.cmake @@ -17,28 +17,31 @@ # # find_package(GLOG) +find_package(PkgConfig QUIET) pkg_check_modules(GLOG_PC libglog) if(GLOG_PC_FOUND) set(GLOG_INCLUDE_DIR "${GLOG_PC_INCLUDEDIR}") list(APPEND GLOG_PC_LIBRARY_DIRS "${GLOG_PC_LIBDIR}") find_library(GLOG_LIB glog PATHS ${GLOG_PC_LIBRARY_DIRS} - PATH_SUFFIXES ${LIB_PATH_SUFFIXES} + PATH_SUFFIXES ${ARROW_LIBRARY_PATH_SUFFIXES} NO_DEFAULT_PATH) elseif(GLOG_ROOT) find_library(GLOG_LIB NAMES glog PATHS ${GLOG_ROOT} - PATH_SUFFIXES ${LIB_PATH_SUFFIXES} + PATH_SUFFIXES ${ARROW_LIBRARY_PATH_SUFFIXES} NO_DEFAULT_PATH) find_path(GLOG_INCLUDE_DIR NAMES glog/logging.h PATHS ${GLOG_ROOT} NO_DEFAULT_PATH - PATH_SUFFIXES ${INCLUDE_PATH_SUFFIXES}) + PATH_SUFFIXES ${ARROW_INCLUDE_PATH_SUFFIXES}) else() - find_library(GLOG_LIB NAMES glog PATH_SUFFIXES ${LIB_PATH_SUFFIXES}) - find_path(GLOG_INCLUDE_DIR NAMES glog/logging.h PATH_SUFFIXES ${INCLUDE_PATH_SUFFIXES}) + find_library(GLOG_LIB NAMES glog PATH_SUFFIXES ${ARROW_LIBRARY_PATH_SUFFIXES}) + find_path(GLOG_INCLUDE_DIR + NAMES glog/logging.h + PATH_SUFFIXES ${ARROW_INCLUDE_PATH_SUFFIXES}) endif() find_package_handle_standard_args(GLOG REQUIRED_VARS GLOG_INCLUDE_DIR GLOG_LIB) diff --git a/cpp/cmake_modules/FindGPerf.cmake b/cpp/cmake_modules/FindGPerf.cmake index 96e507fe75b9..7b9b15a50c71 100644 --- a/cpp/cmake_modules/FindGPerf.cmake +++ b/cpp/cmake_modules/FindGPerf.cmake @@ -35,7 +35,7 @@ SET(GPERF_LIB_SEARCH $ENV{NATIVE_TOOLCHAIN}/gperftools-$ENV{GPERFTOOLS_VERSION}/ FIND_LIBRARY(TCMALLOC_LIB_PATH NAMES libtcmalloc.a PATHS ${GPERF_LIB_SEARCH} - PATH_SUFFIXES ${LIB_PATH_SUFFIXES} + PATH_SUFFIXES ${ARROW_LIBRARY_PATH_SUFFIXES} NO_DEFAULT_PATH ) @@ -52,7 +52,7 @@ ENDIF (TCMALLOC_LIB_PATH AND GOOGLE_PERFTOOLS_INCLUDE_DIR) FIND_LIBRARY(PROFILER_LIB_PATH NAMES libprofiler.a PATHS ${GPERF_LIB_SEARCH} - PATH_SUFFIXES ${LIB_PATH_SUFFIXES} + PATH_SUFFIXES ${ARROW_LIBRARY_PATH_SUFFIXES} ) IF (PROFILER_LIB_PATH AND GOOGLE_PERFTOOLS_INCLUDE_DIR) diff --git a/cpp/cmake_modules/FindGTest.cmake b/cpp/cmake_modules/FindGTest.cmake index 1a22929c9246..8581d921b1cf 100644 --- a/cpp/cmake_modules/FindGTest.cmake +++ b/cpp/cmake_modules/FindGTest.cmake @@ -171,7 +171,7 @@ if(NOT DEFINED GTEST_MSVC_SEARCH) endif() set(_gtest_libpath_suffixes lib) -if(MSVC) +if(MSVC_TOOLCHAIN) if(GTEST_MSVC_SEARCH STREQUAL "MD") list(APPEND _gtest_libpath_suffixes msvc/gtest-md/Debug @@ -194,11 +194,11 @@ find_path(GTEST_INCLUDE_DIR gtest/gtest.h HINTS $ENV{GTEST_ROOT}/include ${GTEST_ROOT}/include - PATH_SUFFIXES ${LIB_PATH_SUFFIXES} + PATH_SUFFIXES ${ARROW_LIBRARY_PATH_SUFFIXES} ) mark_as_advanced(GTEST_INCLUDE_DIR) -if(MSVC AND GTEST_MSVC_SEARCH STREQUAL "MD") +if(MSVC_TOOLCHAIN AND GTEST_MSVC_SEARCH STREQUAL "MD") # The provided /MD project files for Google Test add -md suffixes to the # library names. __gtest_find_library(GTEST_LIBRARY gtest-md gtest) diff --git a/cpp/cmake_modules/FindLz4.cmake b/cpp/cmake_modules/FindLz4.cmake index 8410916432ac..7159f96f70f0 100644 --- a/cpp/cmake_modules/FindLz4.cmake +++ b/cpp/cmake_modules/FindLz4.cmake @@ -15,45 +15,62 @@ # specific language governing permissions and limitations # under the License. -if(MSVC AND NOT DEFINED LZ4_MSVC_STATIC_LIB_SUFFIX) - set(LZ4_MSVC_STATIC_LIB_SUFFIX "_static") +if(MSVC_TOOLCHAIN AND NOT DEFINED LZ4_MSVC_LIB_PREFIX) + set(LZ4_MSVC_LIB_PREFIX "lib") endif() +set(LZ4_LIB_NAME_BASE "${LZ4_MSVC_LIB_PREFIX}lz4") -set(LZ4_STATIC_LIB_SUFFIX "${LZ4_MSVC_STATIC_LIB_SUFFIX}${CMAKE_STATIC_LIBRARY_SUFFIX}") - -set(LZ4_STATIC_LIB_NAME ${CMAKE_STATIC_LIBRARY_PREFIX}lz4${LZ4_STATIC_LIB_SUFFIX}) +if(ARROW_LZ4_USE_SHARED) + set(LZ4_LIB_NAMES) + if(CMAKE_IMPORT_LIBRARY_SUFFIX) + list( + APPEND + LZ4_LIB_NAMES + "${CMAKE_IMPORT_LIBRARY_PREFIX}${LZ4_LIB_NAME_BASE}${CMAKE_IMPORT_LIBRARY_SUFFIX}" + ) + endif() + list( + APPEND + LZ4_LIB_NAMES + "${CMAKE_SHARED_LIBRARY_PREFIX}${LZ4_LIB_NAME_BASE}${CMAKE_SHARED_LIBRARY_SUFFIX}") +else() + if(MSVC AND NOT DEFINED LZ4_MSVC_STATIC_LIB_SUFFIX) + set(LZ4_MSVC_STATIC_LIB_SUFFIX "_static") + endif() + set(LZ4_STATIC_LIB_SUFFIX "${LZ4_MSVC_STATIC_LIB_SUFFIX}${CMAKE_STATIC_LIBRARY_SUFFIX}") + set(LZ4_LIB_NAMES + "${CMAKE_STATIC_LIBRARY_PREFIX}${LZ4_LIB_NAME_BASE}${LZ4_STATIC_LIB_SUFFIX}") +endif() if(LZ4_ROOT) - find_library( - LZ4_LIB - NAMES lz4 ${LZ4_STATIC_LIB_NAME} lib${LZ4_STATIC_LIB_NAME} - "${CMAKE_SHARED_LIBRARY_PREFIX}lz4_static${CMAKE_SHARED_LIBRARY_SUFFIX}" - PATHS ${LZ4_ROOT} - PATH_SUFFIXES ${LIB_PATH_SUFFIXES} - NO_DEFAULT_PATH) + find_library(LZ4_LIB + NAMES ${LZ4_LIB_NAMES} + PATHS ${LZ4_ROOT} + PATH_SUFFIXES ${ARROW_LIBRARY_PATH_SUFFIXES} + NO_DEFAULT_PATH) find_path(LZ4_INCLUDE_DIR NAMES lz4.h PATHS ${LZ4_ROOT} NO_DEFAULT_PATH - PATH_SUFFIXES ${INCLUDE_PATH_SUFFIXES}) + PATH_SUFFIXES ${ARROW_INCLUDE_PATH_SUFFIXES}) else() + find_package(PkgConfig QUIET) pkg_check_modules(LZ4_PC liblz4) if(LZ4_PC_FOUND) set(LZ4_INCLUDE_DIR "${LZ4_PC_INCLUDEDIR}") list(APPEND LZ4_PC_LIBRARY_DIRS "${LZ4_PC_LIBDIR}") - find_library(LZ4_LIB lz4 + find_library(LZ4_LIB + NAMES ${LZ4_LIB_NAMES} PATHS ${LZ4_PC_LIBRARY_DIRS} NO_DEFAULT_PATH - PATH_SUFFIXES ${LIB_PATH_SUFFIXES}) + PATH_SUFFIXES ${ARROW_LIBRARY_PATH_SUFFIXES}) else() - find_library( - LZ4_LIB - NAMES lz4 ${LZ4_STATIC_LIB_NAME} lib${LZ4_STATIC_LIB_NAME} - "${CMAKE_SHARED_LIBRARY_PREFIX}lz4_static${CMAKE_SHARED_LIBRARY_SUFFIX}" - PATH_SUFFIXES ${LIB_PATH_SUFFIXES}) - find_path(LZ4_INCLUDE_DIR NAMES lz4.h PATH_SUFFIXES ${INCLUDE_PATH_SUFFIXES}) + find_library(LZ4_LIB + NAMES ${LZ4_LIB_NAMES} + PATH_SUFFIXES ${ARROW_LIBRARY_PATH_SUFFIXES}) + find_path(LZ4_INCLUDE_DIR NAMES lz4.h PATH_SUFFIXES ${ARROW_INCLUDE_PATH_SUFFIXES}) endif() endif() diff --git a/cpp/cmake_modules/FindORC.cmake b/cpp/cmake_modules/FindORC.cmake index 1cda71fee0ac..1be149c93b2a 100644 --- a/cpp/cmake_modules/FindORC.cmake +++ b/cpp/cmake_modules/FindORC.cmake @@ -26,17 +26,17 @@ if(ORC_ROOT) NAMES orc PATHS ${ORC_ROOT} NO_DEFAULT_PATH - PATH_SUFFIXES ${LIB_PATH_SUFFIXES}) + PATH_SUFFIXES ${ARROW_LIBRARY_PATH_SUFFIXES}) find_path(ORC_INCLUDE_DIR NAMES orc/orc-config.hh PATHS ${ORC_ROOT} NO_DEFAULT_PATH - PATH_SUFFIXES ${INCLUDE_PATH_SUFFIXES}) + PATH_SUFFIXES ${ARROW_INCLUDE_PATH_SUFFIXES}) else() - find_library(ORC_STATIC_LIB NAMES orc PATH_SUFFIXES ${LIB_PATH_SUFFIXES}) + find_library(ORC_STATIC_LIB NAMES orc PATH_SUFFIXES ${ARROW_LIBRARY_PATH_SUFFIXES}) find_path(ORC_INCLUDE_DIR NAMES orc/orc-config.hh - PATH_SUFFIXES ${INCLUDE_PATH_SUFFIXES}) + PATH_SUFFIXES ${ARROW_INCLUDE_PATH_SUFFIXES}) endif() if(ORC_STATIC_LIB AND ORC_INCLUDE_DIR) diff --git a/cpp/cmake_modules/FindRE2.cmake b/cpp/cmake_modules/FindRE2.cmake index b8d76cef1947..645a20f7c09c 100644 --- a/cpp/cmake_modules/FindRE2.cmake +++ b/cpp/cmake_modules/FindRE2.cmake @@ -15,6 +15,7 @@ # specific language governing permissions and limitations # under the License. +find_package(PkgConfig QUIET) pkg_check_modules(RE2_PC re2) if(RE2_PC_FOUND) set(RE2_INCLUDE_DIR "${RE2_PC_INCLUDEDIR}") @@ -22,7 +23,7 @@ if(RE2_PC_FOUND) list(APPEND RE2_PC_LIBRARY_DIRS "${RE2_PC_LIBDIR}") find_library(RE2_LIB re2 PATHS ${RE2_PC_LIBRARY_DIRS} - PATH_SUFFIXES ${LIB_PATH_SUFFIXES} + PATH_SUFFIXES ${ARROW_LIBRARY_PATH_SUFFIXES} NO_DEFAULT_PATH) # On Fedora, the reported prefix is wrong. As users likely run into this, @@ -41,13 +42,13 @@ elseif(RE2_ROOT) "${CMAKE_STATIC_LIBRARY_PREFIX}re2${RE2_MSVC_STATIC_LIB_SUFFIX}${CMAKE_STATIC_LIBRARY_SUFFIX}" "${CMAKE_SHARED_LIBRARY_PREFIX}re2${CMAKE_SHARED_LIBRARY_SUFFIX}" PATHS ${RE2_ROOT} - PATH_SUFFIXES ${LIB_PATH_SUFFIXES} + PATH_SUFFIXES ${ARROW_LIBRARY_PATH_SUFFIXES} NO_DEFAULT_PATH) find_path(RE2_INCLUDE_DIR NAMES re2/re2.h PATHS ${RE2_ROOT} NO_DEFAULT_PATH - PATH_SUFFIXES ${INCLUDE_PATH_SUFFIXES}) + PATH_SUFFIXES ${ARROW_INCLUDE_PATH_SUFFIXES}) else() find_library( RE2_LIB @@ -55,15 +56,24 @@ else() re2_static re2 "${CMAKE_STATIC_LIBRARY_PREFIX}re2${RE2_MSVC_STATIC_LIB_SUFFIX}${CMAKE_STATIC_LIBRARY_SUFFIX}" "${CMAKE_SHARED_LIBRARY_PREFIX}re2${CMAKE_SHARED_LIBRARY_SUFFIX}" - PATH_SUFFIXES ${LIB_PATH_SUFFIXES}) - find_path(RE2_INCLUDE_DIR NAMES re2/re2.h PATH_SUFFIXES ${INCLUDE_PATH_SUFFIXES}) + PATH_SUFFIXES ${ARROW_LIBRARY_PATH_SUFFIXES}) + find_path(RE2_INCLUDE_DIR NAMES re2/re2.h PATH_SUFFIXES ${ARROW_INCLUDE_PATH_SUFFIXES}) endif() find_package_handle_standard_args(RE2 REQUIRED_VARS RE2_LIB RE2_INCLUDE_DIR) if(RE2_FOUND) - add_library(RE2::re2 UNKNOWN IMPORTED) - set_target_properties(RE2::re2 - PROPERTIES IMPORTED_LOCATION "${RE2_LIB}" - INTERFACE_INCLUDE_DIRECTORIES "${RE2_INCLUDE_DIR}") + if(NOT TARGET RE2::re2) + add_library(RE2::re2 UNKNOWN IMPORTED) + set_target_properties(RE2::re2 + PROPERTIES IMPORTED_LOCATION "${RE2_LIB}" + INTERFACE_INCLUDE_DIRECTORIES "${RE2_INCLUDE_DIR}") + endif() + # Some third-party dependencies (namely gRPC) are on the look-out for a lower-case re2 Target. + if(NOT TARGET re2::re2) + add_library(re2::re2 UNKNOWN IMPORTED) + set_target_properties(re2::re2 + PROPERTIES IMPORTED_LOCATION "${RE2_LIB}" + INTERFACE_INCLUDE_DIRECTORIES "${RE2_INCLUDE_DIR}") + endif() endif() diff --git a/cpp/cmake_modules/FindSnappyAlt.cmake b/cpp/cmake_modules/FindSnappy.cmake similarity index 53% rename from cpp/cmake_modules/FindSnappyAlt.cmake rename to cpp/cmake_modules/FindSnappy.cmake index 019c8794f18a..24fdf1031c7a 100644 --- a/cpp/cmake_modules/FindSnappyAlt.cmake +++ b/cpp/cmake_modules/FindSnappy.cmake @@ -15,25 +15,42 @@ # specific language governing permissions and limitations # under the License. +if(ARROW_SNAPPY_USE_SHARED) + set(SNAPPY_LIB_NAMES) + if(CMAKE_IMPORT_LIBRARY_SUFFIX) + list(APPEND SNAPPY_LIB_NAMES + "${CMAKE_IMPORT_LIBRARY_PREFIX}snappy${CMAKE_IMPORT_LIBRARY_SUFFIX}") + endif() + list(APPEND SNAPPY_LIB_NAMES + "${CMAKE_SHARED_LIBRARY_PREFIX}snappy${CMAKE_SHARED_LIBRARY_SUFFIX}") +else() + set(SNAPPY_STATIC_LIB_NAME_BASE "snappy") + if(MSVC) + set(SNAPPY_STATIC_LIB_NAME_BASE "${SNAPPY_STATIC_LIB_NAME_BASE}${SNAPPY_MSVC_STATIC_LIB_SUFFIX}") + endif() + set(SNAPPY_LIB_NAMES "${CMAKE_STATIC_LIBRARY_PREFIX}${SNAPPY_STATIC_LIB_NAME_BASE}${CMAKE_STATIC_LIBRARY_SUFFIX}") +endif() + if(Snappy_ROOT) find_library(Snappy_LIB - NAMES snappy + NAMES ${SNAPPY_LIB_NAMES} PATHS ${Snappy_ROOT} - PATH_SUFFIXES ${LIB_PATH_SUFFIXES} + PATH_SUFFIXES ${ARROW_LIBRARY_PATH_SUFFIXES} NO_DEFAULT_PATH) find_path(Snappy_INCLUDE_DIR NAMES snappy.h PATHS ${Snappy_ROOT} NO_DEFAULT_PATH - PATH_SUFFIXES ${INCLUDE_PATH_SUFFIXES}) + PATH_SUFFIXES ${ARROW_INCLUDE_PATH_SUFFIXES}) else() - find_library(Snappy_LIB NAMES snappy) - find_path(Snappy_INCLUDE_DIR NAMES snappy.h PATH_SUFFIXES ${INCLUDE_PATH_SUFFIXES}) + find_library(Snappy_LIB NAMES ${SNAPPY_LIB_NAMES}) + find_path(Snappy_INCLUDE_DIR NAMES snappy.h PATH_SUFFIXES ${ARROW_INCLUDE_PATH_SUFFIXES}) endif() -find_package_handle_standard_args(SnappyAlt REQUIRED_VARS Snappy_LIB Snappy_INCLUDE_DIR) +find_package_handle_standard_args(Snappy REQUIRED_VARS Snappy_LIB Snappy_INCLUDE_DIR) -if(SnappyAlt_FOUND) +# CMake 3.2 does uppercase the FOUND variable +if(Snappy_FOUND OR SNAPPY_FOUND) add_library(Snappy::snappy UNKNOWN IMPORTED) set_target_properties(Snappy::snappy PROPERTIES IMPORTED_LOCATION "${Snappy_LIB}" diff --git a/cpp/cmake_modules/FindThrift.cmake b/cpp/cmake_modules/FindThrift.cmake index bb3eb5608592..273d907ed07a 100644 --- a/cpp/cmake_modules/FindThrift.cmake +++ b/cpp/cmake_modules/FindThrift.cmake @@ -24,7 +24,7 @@ # This module defines # THRIFT_VERSION, version string of ant if found # THRIFT_INCLUDE_DIR, where to find THRIFT headers -# THRIFT_STATIC_LIB, THRIFT static library +# THRIFT_LIB, THRIFT library # THRIFT_FOUND, If false, do not try to use ant function(EXTRACT_THRIFT_VERSION) @@ -39,12 +39,40 @@ function(EXTRACT_THRIFT_VERSION) endif() endfunction(EXTRACT_THRIFT_VERSION) -if(MSVC AND NOT THRIFT_MSVC_STATIC_LIB_SUFFIX) - set(THRIFT_MSVC_STATIC_LIB_SUFFIX md) +if(MSVC_TOOLCHAIN AND NOT DEFINED THRIFT_MSVC_LIB_SUFFIX) + if(NOT ARROW_THRIFT_USE_SHARED) + if(ARROW_USE_STATIC_CRT) + set(THRIFT_MSVC_LIB_SUFFIX "mt") + else() + set(THRIFT_MSVC_LIB_SUFFIX "md") + endif() + endif() +endif() +set(THRIFT_LIB_NAME_BASE "thrift${THRIFT_MSVC_LIB_SUFFIX}") + +if(ARROW_THRIFT_USE_SHARED) + set(THRIFT_LIB_NAMES thrift) + if(CMAKE_IMPORT_LIBRARY_SUFFIX) + list( + APPEND + THRIFT_LIB_NAMES + "${CMAKE_IMPORT_LIBRARY_PREFIX}${THRIFT_LIB_NAME_BASE}${CMAKE_IMPORT_LIBRARY_SUFFIX}" + ) + endif() + list( + APPEND + THRIFT_LIB_NAMES + "${CMAKE_SHARED_LIBRARY_PREFIX}${THRIFT_LIB_NAME_BASE}${CMAKE_SHARED_LIBRARY_SUFFIX}" + ) +else() + set( + THRIFT_LIB_NAMES + "${CMAKE_STATIC_LIBRARY_PREFIX}${THRIFT_LIB_NAME_BASE}${CMAKE_STATIC_LIBRARY_SUFFIX}") endif() if(Thrift_ROOT) - find_library(THRIFT_STATIC_LIB thrift${THRIFT_MSVC_STATIC_LIB_SUFFIX} + find_library(THRIFT_LIB + NAMES ${THRIFT_LIB_NAMES} PATHS ${Thrift_ROOT} PATH_SUFFIXES "lib/${CMAKE_LIBRARY_ARCHITECTURE}" "lib") find_path(THRIFT_INCLUDE_DIR thrift/Thrift.h @@ -55,13 +83,15 @@ if(Thrift_ROOT) else() # THRIFT-4760: The pkgconfig files are currently only installed when using autotools. # Starting with 0.13, they are also installed for the CMake-based installations of Thrift. + find_package(PkgConfig QUIET) pkg_check_modules(THRIFT_PC thrift) if(THRIFT_PC_FOUND) set(THRIFT_INCLUDE_DIR "${THRIFT_PC_INCLUDEDIR}") list(APPEND THRIFT_PC_LIBRARY_DIRS "${THRIFT_PC_LIBDIR}") - find_library(THRIFT_STATIC_LIB thrift${THRIFT_MSVC_STATIC_LIB_SUFFIX} + find_library(THRIFT_LIB + NAMES ${THRIFT_LIB_NAMES} PATHS ${THRIFT_PC_LIBRARY_DIRS} NO_DEFAULT_PATH) find_program(THRIFT_COMPILER thrift @@ -70,7 +100,8 @@ else() PATH_SUFFIXES "bin") set(THRIFT_VERSION ${THRIFT_PC_VERSION}) else() - find_library(THRIFT_STATIC_LIB thrift${THRIFT_MSVC_STATIC_LIB_SUFFIX} + find_library(THRIFT_LIB + NAMES ${THRIFT_LIB_NAMES} PATH_SUFFIXES "lib/${CMAKE_LIBRARY_ARCHITECTURE}" "lib") find_path(THRIFT_INCLUDE_DIR thrift/Thrift.h PATH_SUFFIXES "include") find_program(THRIFT_COMPILER thrift PATH_SUFFIXES "bin") @@ -86,7 +117,7 @@ endif() find_package_handle_standard_args(Thrift REQUIRED_VARS - THRIFT_STATIC_LIB + THRIFT_LIB THRIFT_INCLUDE_DIR VERSION_VAR THRIFT_VERSION @@ -94,11 +125,15 @@ find_package_handle_standard_args(Thrift if(Thrift_FOUND OR THRIFT_FOUND) set(Thrift_FOUND TRUE) - add_library(thrift::thrift STATIC IMPORTED) + if(ARROW_THRIFT_USE_SHARED) + add_library(thrift::thrift SHARED IMPORTED) + else() + add_library(thrift::thrift STATIC IMPORTED) + endif() set_target_properties(thrift::thrift - PROPERTIES IMPORTED_LOCATION "${THRIFT_STATIC_LIB}" + PROPERTIES IMPORTED_LOCATION "${THRIFT_LIB}" INTERFACE_INCLUDE_DIRECTORIES "${THRIFT_INCLUDE_DIR}") - if(WIN32 AND NOT MSVC) + if(WIN32 AND NOT MSVC_TOOLCHAIN) # We don't need this for Visual C++ because Thrift uses # "#pragma comment(lib, "Ws2_32.lib")" in # thrift/windows/config.h for Visual C++. diff --git a/cpp/cmake_modules/FindgRPCAlt.cmake b/cpp/cmake_modules/FindgRPCAlt.cmake index 35b36827f83b..d6ba331aea39 100644 --- a/cpp/cmake_modules/FindgRPCAlt.cmake +++ b/cpp/cmake_modules/FindgRPCAlt.cmake @@ -61,27 +61,27 @@ if(gRPC_ROOT) find_library(GRPC_GPR_LIB NAMES ${GRPC_GPR_LIB_NAMES} PATHS ${gRPC_ROOT} - PATH_SUFFIXES ${LIB_PATH_SUFFIXES} + PATH_SUFFIXES ${ARROW_LIBRARY_PATH_SUFFIXES} NO_DEFAULT_PATH) find_library(GRPC_GRPC_LIB NAMES ${GRPC_GRPC_LIB_NAMES} PATHS ${gRPC_ROOT} - PATH_SUFFIXES ${LIB_PATH_SUFFIXES} + PATH_SUFFIXES ${ARROW_LIBRARY_PATH_SUFFIXES} NO_DEFAULT_PATH) find_library(GRPC_GRPCPP_LIB NAMES ${GRPC_GRPCPP_LIB_NAMES} PATHS ${gRPC_ROOT} - PATH_SUFFIXES ${LIB_PATH_SUFFIXES} + PATH_SUFFIXES ${ARROW_LIBRARY_PATH_SUFFIXES} NO_DEFAULT_PATH) find_library(GRPC_ADDRESS_SORTING_LIB NAMES ${GRPC_ADDRESS_SORTING_LIB_NAMES} PATHS ${gRPC_ROOT} - PATH_SUFFIXES ${LIB_PATH_SUFFIXES} + PATH_SUFFIXES ${ARROW_LIBRARY_PATH_SUFFIXES} NO_DEFAULT_PATH) find_library(GRPC_UPB_LIB NAMES ${GRPC_UPB_LIB_NAMES} PATHS ${gRPC_ROOT} - PATH_SUFFIXES ${LIB_PATH_SUFFIXES} + PATH_SUFFIXES ${ARROW_LIBRARY_PATH_SUFFIXES} NO_DEFAULT_PATH) find_program(GRPC_CPP_PLUGIN grpc_cpp_plugin NO_DEFAULT_PATH PATHS ${gRPC_ROOT} @@ -90,8 +90,9 @@ if(gRPC_ROOT) NAMES grpc/grpc.h PATHS ${gRPC_ROOT} NO_DEFAULT_PATH - PATH_SUFFIXES ${INCLUDE_PATH_SUFFIXES}) + PATH_SUFFIXES ${ARROW_INCLUDE_PATH_SUFFIXES}) else() + find_package(PkgConfig QUIET) pkg_check_modules(GRPC_PC grpc++) if(GRPC_PC_FOUND) set(GRPC_ALT_VERSION "${GRPC_PC_VERSION}") @@ -102,27 +103,27 @@ else() find_library(GRPC_GPR_LIB NAMES ${GRPC_GPR_LIB_NAMES} PATHS ${GRPC_PC_LIBRARY_DIRS} - PATH_SUFFIXES ${LIB_PATH_SUFFIXES} + PATH_SUFFIXES ${ARROW_LIBRARY_PATH_SUFFIXES} NO_DEFAULT_PATH) find_library(GRPC_GRPC_LIB NAMES ${GRPC_GRPC_LIB_NAMES} PATHS ${GRPC_PC_LIBRARY_DIRS} - PATH_SUFFIXES ${LIB_PATH_SUFFIXES} + PATH_SUFFIXES ${ARROW_LIBRARY_PATH_SUFFIXES} NO_DEFAULT_PATH) find_library(GRPC_GRPCPP_LIB NAMES ${GRPC_GRPCPP_LIB_NAMES} PATHS ${GRPC_PC_LIBRARY_DIRS} - PATH_SUFFIXES ${LIB_PATH_SUFFIXES} + PATH_SUFFIXES ${ARROW_LIBRARY_PATH_SUFFIXES} NO_DEFAULT_PATH) find_library(GRPC_ADDRESS_SORTING_LIB NAMES ${GRPC_ADDRESS_SORTING_LIB_NAMES} PATHS ${GRPC_PC_LIBRARY_DIRS} - PATH_SUFFIXES ${LIB_PATH_SUFFIXES} + PATH_SUFFIXES ${ARROW_LIBRARY_PATH_SUFFIXES} NO_DEFAULT_PATH) find_library(GRPC_UPB_LIB NAMES ${GRPC_UPB_LIB_NAMES} PATHS ${GRPC_PC_LIBRARY_DIRS} - PATH_SUFFIXES ${LIB_PATH_SUFFIXES} + PATH_SUFFIXES ${ARROW_LIBRARY_PATH_SUFFIXES} NO_DEFAULT_PATH) find_program(GRPC_CPP_PLUGIN grpc_cpp_plugin HINTS ${GRPC_PC_PREFIX} @@ -131,21 +132,23 @@ else() else() find_library(GRPC_GPR_LIB NAMES ${GRPC_GPR_LIB_NAMES} - PATH_SUFFIXES ${LIB_PATH_SUFFIXES}) + PATH_SUFFIXES ${ARROW_LIBRARY_PATH_SUFFIXES}) find_library(GRPC_GRPC_LIB NAMES ${GRPC_GRPC_LIB_NAMES} - PATH_SUFFIXES ${LIB_PATH_SUFFIXES}) + PATH_SUFFIXES ${ARROW_LIBRARY_PATH_SUFFIXES}) find_library(GRPC_GRPCPP_LIB NAMES ${GRPC_GRPCPP_LIB_NAMES} - PATH_SUFFIXES ${LIB_PATH_SUFFIXES}) + PATH_SUFFIXES ${ARROW_LIBRARY_PATH_SUFFIXES}) find_library(GRPC_ADDRESS_SORTING_LIB NAMES ${GRPC_ADDRESS_SORTING_LIB_NAMES} - PATH_SUFFIXES ${LIB_PATH_SUFFIXES}) + PATH_SUFFIXES ${ARROW_LIBRARY_PATH_SUFFIXES}) find_library(GRPC_UPB_LIB NAMES ${GRPC_UPB_LIB_NAMES} - PATH_SUFFIXES ${LIB_PATH_SUFFIXES}) + PATH_SUFFIXES ${ARROW_LIBRARY_PATH_SUFFIXES}) find_program(GRPC_CPP_PLUGIN grpc_cpp_plugin PATH_SUFFIXES "bin") - find_path(GRPC_INCLUDE_DIR NAMES grpc/grpc.h PATH_SUFFIXES ${INCLUDE_PATH_SUFFIXES}) + find_path(GRPC_INCLUDE_DIR + NAMES grpc/grpc.h + PATH_SUFFIXES ${ARROW_INCLUDE_PATH_SUFFIXES}) endif() endif() diff --git a/cpp/cmake_modules/FindgflagsAlt.cmake b/cpp/cmake_modules/FindgflagsAlt.cmake index 556048c6076f..cf6d2654f73a 100644 --- a/cpp/cmake_modules/FindgflagsAlt.cmake +++ b/cpp/cmake_modules/FindgflagsAlt.cmake @@ -21,18 +21,18 @@ if(gflags_ROOT) find_library(gflags_LIB NAMES gflags PATHS ${gflags_ROOT} - PATH_SUFFIXES ${LIB_PATH_SUFFIXES} + PATH_SUFFIXES ${ARROW_LIBRARY_PATH_SUFFIXES} NO_DEFAULT_PATH) find_path(GFLAGS_INCLUDE_DIR NAMES gflags/gflags.h PATHS ${gflags_ROOT} NO_DEFAULT_PATH - PATH_SUFFIXES ${INCLUDE_PATH_SUFFIXES}) + PATH_SUFFIXES ${ARROW_INCLUDE_PATH_SUFFIXES}) else() find_library(gflags_LIB NAMES gflags) find_path(GFLAGS_INCLUDE_DIR NAMES gflags/gflags.h - PATH_SUFFIXES ${INCLUDE_PATH_SUFFIXES}) + PATH_SUFFIXES ${ARROW_INCLUDE_PATH_SUFFIXES}) endif() find_package_handle_standard_args(gflagsAlt REQUIRED_VARS gflags_LIB GFLAGS_INCLUDE_DIR) diff --git a/cpp/cmake_modules/Findutf8proc.cmake b/cpp/cmake_modules/Findutf8proc.cmake index d3063827add9..e3ae0ca87f1f 100644 --- a/cpp/cmake_modules/Findutf8proc.cmake +++ b/cpp/cmake_modules/Findutf8proc.cmake @@ -16,52 +16,51 @@ # under the License. if(ARROW_UTF8PROC_USE_SHARED) - set(UTF8PROC_LIB_NAMES) + set(utf8proc_LIB_NAMES) if(CMAKE_IMPORT_LIBRARY_SUFFIX) - list(APPEND UTF8PROC_LIB_NAMES + list(APPEND utf8proc_LIB_NAMES "${CMAKE_IMPORT_LIBRARY_PREFIX}utf8proc${CMAKE_IMPORT_LIBRARY_SUFFIX}") endif() - list(APPEND UTF8PROC_LIB_NAMES + list(APPEND utf8proc_LIB_NAMES "${CMAKE_SHARED_LIBRARY_PREFIX}utf8proc${CMAKE_SHARED_LIBRARY_SUFFIX}") else() - if(MSVC AND NOT DEFINED UTF8PROC_MSVC_STATIC_LIB_SUFFIX) - set(UTF8PROC_MSVC_STATIC_LIB_SUFFIX "_static") + if(MSVC AND NOT DEFINED utf8proc_MSVC_STATIC_LIB_SUFFIX) + set(utf8proc_MSVC_STATIC_LIB_SUFFIX "_static") endif() - set(UTF8PROC_STATIC_LIB_SUFFIX - "${UTF8PROC_MSVC_STATIC_LIB_SUFFIX}${CMAKE_STATIC_LIBRARY_SUFFIX}") - set(UTF8PROC_STATIC_LIB_NAME ${CMAKE_STATIC_LIBRARY_PREFIX}utf8proc${UTF8PROC_STATIC_LIB_SUFFIX}) - set(UTF8PROC_LIB_NAMES "${UTF8PROC_STATIC_LIB_NAME}" "lib${UTF8PROC_STATIC_LIB_NAME}") + set(utf8proc_STATIC_LIB_SUFFIX + "${utf8proc_MSVC_STATIC_LIB_SUFFIX}${CMAKE_STATIC_LIBRARY_SUFFIX}") + set(utf8proc_LIB_NAMES "${CMAKE_STATIC_LIBRARY_PREFIX}utf8proc${utf8proc_STATIC_LIB_SUFFIX}") endif() if(utf8proc_ROOT) find_library( - UTF8PROC_LIB - NAMES ${UTF8PROC_LIB_NAMES} + utf8proc_LIB + NAMES ${utf8proc_LIB_NAMES} PATHS ${utf8proc_ROOT} - PATH_SUFFIXES ${LIB_PATH_SUFFIXES} + PATH_SUFFIXES ${ARROW_LIBRARY_PATH_SUFFIXES} NO_DEFAULT_PATH) - find_path(UTF8PROC_INCLUDE_DIR + find_path(utf8proc_INCLUDE_DIR NAMES utf8proc.h PATHS ${utf8proc_ROOT} NO_DEFAULT_PATH - PATH_SUFFIXES ${INCLUDE_PATH_SUFFIXES}) + PATH_SUFFIXES ${ARROW_INCLUDE_PATH_SUFFIXES}) else() find_library( - UTF8PROC_LIB - NAMES ${UTF8PROC_LIB_NAMES} - PATH_SUFFIXES ${LIB_PATH_SUFFIXES}) - find_path(UTF8PROC_INCLUDE_DIR NAMES utf8proc.h PATH_SUFFIXES ${INCLUDE_PATH_SUFFIXES}) + utf8proc_LIB + NAMES ${utf8proc_LIB_NAMES} + PATH_SUFFIXES ${ARROW_LIBRARY_PATH_SUFFIXES}) + find_path(utf8proc_INCLUDE_DIR NAMES utf8proc.h PATH_SUFFIXES ${ARROW_INCLUDE_PATH_SUFFIXES}) endif() -find_package_handle_standard_args(utf8proc REQUIRED_VARS UTF8PROC_LIB UTF8PROC_INCLUDE_DIR) +find_package_handle_standard_args(utf8proc REQUIRED_VARS utf8proc_LIB utf8proc_INCLUDE_DIR) # CMake 3.2 does uppercase the FOUND variable if(UTF8PROC_FOUND OR utf8proc_FOUND) set(utf8proc_FOUND TRUE) add_library(utf8proc::utf8proc UNKNOWN IMPORTED) set_target_properties(utf8proc::utf8proc - PROPERTIES IMPORTED_LOCATION "${UTF8PROC_LIB}" - INTERFACE_INCLUDE_DIRECTORIES "${UTF8PROC_INCLUDE_DIR}") + PROPERTIES IMPORTED_LOCATION "${utf8proc_LIB}" + INTERFACE_INCLUDE_DIRECTORIES "${utf8proc_INCLUDE_DIR}") if(NOT ARROW_UTF8PROC_USE_SHARED) set_target_properties(utf8proc::utf8proc PROPERTIES INTERFACE_COMPILER_DEFINITIONS "UTF8PROC_STATIC") diff --git a/cpp/cmake_modules/FindZSTD.cmake b/cpp/cmake_modules/Findzstd.cmake similarity index 64% rename from cpp/cmake_modules/FindZSTD.cmake rename to cpp/cmake_modules/Findzstd.cmake index 84d21d2b5c73..6659a682da7e 100644 --- a/cpp/cmake_modules/FindZSTD.cmake +++ b/cpp/cmake_modules/Findzstd.cmake @@ -15,22 +15,32 @@ # specific language governing permissions and limitations # under the License. +if(MSVC AND NOT DEFINED ZSTD_MSVC_LIB_PREFIX) + set(ZSTD_MSVC_LIB_PREFIX "lib") +endif() +set(ZSTD_LIB_NAME_BASE "${ZSTD_MSVC_LIB_PREFIX}zstd") + if(ARROW_ZSTD_USE_SHARED) set(ZSTD_LIB_NAMES) if(CMAKE_IMPORT_LIBRARY_SUFFIX) - list(APPEND ZSTD_LIB_NAMES - "${CMAKE_IMPORT_LIBRARY_PREFIX}zstd${CMAKE_IMPORT_LIBRARY_SUFFIX}") + list( + APPEND + ZSTD_LIB_NAMES + "${CMAKE_IMPORT_LIBRARY_PREFIX}${ZSTD_LIB_NAME_BASE}${CMAKE_IMPORT_LIBRARY_SUFFIX}" + ) endif() - list(APPEND ZSTD_LIB_NAMES - "${CMAKE_SHARED_LIBRARY_PREFIX}zstd${CMAKE_SHARED_LIBRARY_SUFFIX}") + list( + APPEND + ZSTD_LIB_NAMES + "${CMAKE_SHARED_LIBRARY_PREFIX}${ZSTD_LIB_NAME_BASE}${CMAKE_SHARED_LIBRARY_SUFFIX}") else() if(MSVC AND NOT DEFINED ZSTD_MSVC_STATIC_LIB_SUFFIX) set(ZSTD_MSVC_STATIC_LIB_SUFFIX "_static") endif() set(ZSTD_STATIC_LIB_SUFFIX "${ZSTD_MSVC_STATIC_LIB_SUFFIX}${CMAKE_STATIC_LIBRARY_SUFFIX}") - set(ZSTD_STATIC_LIB_NAME ${CMAKE_STATIC_LIBRARY_PREFIX}zstd${ZSTD_STATIC_LIB_SUFFIX}) - set(ZSTD_LIB_NAMES "${ZSTD_STATIC_LIB_NAME}" "lib${ZSTD_STATIC_LIB_NAME}") + set(ZSTD_LIB_NAMES + "${CMAKE_STATIC_LIBRARY_PREFIX}${ZSTD_LIB_NAME_BASE}${ZSTD_STATIC_LIB_SUFFIX}") endif() # First, find via if specified ZTD_ROOT @@ -39,35 +49,40 @@ if(ZSTD_ROOT) find_library(ZSTD_LIB NAMES ${ZSTD_LIB_NAMES} PATHS ${ZSTD_ROOT} - PATH_SUFFIXES ${LIB_PATH_SUFFIXES} + PATH_SUFFIXES ${ARROW_LIBRARY_PATH_SUFFIXES} NO_DEFAULT_PATH) find_path(ZSTD_INCLUDE_DIR NAMES zstd.h PATHS ${ZSTD_ROOT} NO_DEFAULT_PATH - PATH_SUFFIXES ${INCLUDE_PATH_SUFFIXES}) + PATH_SUFFIXES ${ARROW_INCLUDE_PATH_SUFFIXES}) else() # Second, find via pkg_check_modules + find_package(PkgConfig QUIET) pkg_check_modules(ZSTD_PC libzstd) if(ZSTD_PC_FOUND) set(ZSTD_INCLUDE_DIR "${ZSTD_PC_INCLUDEDIR}") list(APPEND ZSTD_PC_LIBRARY_DIRS "${ZSTD_PC_LIBDIR}") - find_library(ZSTD_LIB zstd + find_library(ZSTD_LIB + NAMES ${ZSTD_LIB_NAMES} PATHS ${ZSTD_PC_LIBRARY_DIRS} NO_DEFAULT_PATH - PATH_SUFFIXES ${LIB_PATH_SUFFIXES}) - # Third, check all other CMake paths + PATH_SUFFIXES ${ARROW_LIBRARY_PATH_SUFFIXES}) else() - find_library(ZSTD_LIB NAMES ${ZSTD_LIB_NAMES} PATH_SUFFIXES ${LIB_PATH_SUFFIXES}) - find_path(ZSTD_INCLUDE_DIR NAMES zstd.h PATH_SUFFIXES ${INCLUDE_PATH_SUFFIXES}) + # Third, check all other CMake paths + find_library(ZSTD_LIB + NAMES ${ZSTD_LIB_NAMES} + PATH_SUFFIXES ${ARROW_LIBRARY_PATH_SUFFIXES}) + find_path(ZSTD_INCLUDE_DIR NAMES zstd.h PATH_SUFFIXES ${ARROW_INCLUDE_PATH_SUFFIXES}) endif() endif() -find_package_handle_standard_args(ZSTD REQUIRED_VARS ZSTD_LIB ZSTD_INCLUDE_DIR) +find_package_handle_standard_args(zstd REQUIRED_VARS ZSTD_LIB ZSTD_INCLUDE_DIR) -if(ZSTD_FOUND) +# CMake 3.2 does uppercase the FOUND variable +if(zstd_FOUND OR ZSTD_FOUND) add_library(zstd::libzstd UNKNOWN IMPORTED) set_target_properties(zstd::libzstd PROPERTIES IMPORTED_LOCATION "${ZSTD_LIB}" diff --git a/cpp/cmake_modules/SetupCxxFlags.cmake b/cpp/cmake_modules/SetupCxxFlags.cmake index 32acc3e54af1..1606199c4064 100644 --- a/cpp/cmake_modules/SetupCxxFlags.cmake +++ b/cpp/cmake_modules/SetupCxxFlags.cmake @@ -46,9 +46,13 @@ if(ARROW_CPU_FLAG STREQUAL "x86") set(CXX_SUPPORTS_SSE4_2 TRUE) else() set(ARROW_SSE4_2_FLAG "-msse4.2") - set(ARROW_AVX2_FLAG "-mavx2") + set(ARROW_AVX2_FLAG "-march=haswell") # skylake-avx512 consists of AVX512F,AVX512BW,AVX512VL,AVX512CD,AVX512DQ set(ARROW_AVX512_FLAG "-march=skylake-avx512 -mbmi2") + # Append the avx2/avx512 subset option also, fix issue ARROW-9877 for homebrew-cpp + set(ARROW_AVX2_FLAG "${ARROW_AVX2_FLAG} -mavx2") + set(ARROW_AVX512_FLAG + "${ARROW_AVX512_FLAG} -mavx512f -mavx512cd -mavx512vl -mavx512dq -mavx512bw") check_cxx_compiler_flag(${ARROW_SSE4_2_FLAG} CXX_SUPPORTS_SSE4_2) endif() check_cxx_compiler_flag(${ARROW_AVX2_FLAG} CXX_SUPPORTS_AVX2) @@ -59,14 +63,15 @@ if(ARROW_CPU_FLAG STREQUAL "x86") check_cxx_compiler_flag(${ARROW_AVX512_FLAG} CXX_SUPPORTS_AVX512) endif() # Runtime SIMD level it can get from compiler - if(CXX_SUPPORTS_SSE4_2) + if(CXX_SUPPORTS_SSE4_2 + AND ARROW_RUNTIME_SIMD_LEVEL MATCHES "^(SSE4_2|AVX2|AVX512|MAX)$") add_definitions(-DARROW_HAVE_RUNTIME_SSE4_2) endif() - if(CXX_SUPPORTS_AVX2) - add_definitions(-DARROW_HAVE_RUNTIME_AVX2) + if(CXX_SUPPORTS_AVX2 AND ARROW_RUNTIME_SIMD_LEVEL MATCHES "^(AVX2|AVX512|MAX)$") + add_definitions(-DARROW_HAVE_RUNTIME_AVX2 -DARROW_HAVE_RUNTIME_BMI2) endif() - if(CXX_SUPPORTS_AVX512) - add_definitions(-DARROW_HAVE_RUNTIME_AVX512) + if(CXX_SUPPORTS_AVX512 AND ARROW_RUNTIME_SIMD_LEVEL MATCHES "^(AVX512|MAX)$") + add_definitions(-DARROW_HAVE_RUNTIME_AVX512 -DARROW_HAVE_RUNTIME_BMI2) endif() elseif(ARROW_CPU_FLAG STREQUAL "ppc") # power compiler flags, gcc/clang only @@ -283,6 +288,13 @@ elseif(CMAKE_CXX_COMPILER_ID STREQUAL "GNU") set(CXX_ONLY_FLAGS "${CXX_ONLY_FLAGS} -Wno-noexcept-type") endif() + if(CMAKE_CXX_COMPILER_VERSION VERSION_GREATER "5.2") + # Disabling semantic interposition allows faster calling conventions + # when calling global functions internally, and can also help inlining. + # See https://stackoverflow.com/questions/35745543/new-option-in-gcc-5-3-fno-semantic-interposition + set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -fno-semantic-interposition") + endif() + if(CMAKE_CXX_COMPILER_VERSION VERSION_GREATER "4.9") # Add colors when paired with ninja set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fdiagnostics-color=always") @@ -533,12 +545,6 @@ else() message(FATAL_ERROR "Unknown build type: ${CMAKE_BUILD_TYPE}") endif() -if("${CMAKE_CXX_FLAGS}" MATCHES "-DNDEBUG") - set(ARROW_DEFINITION_FLAGS "-DNDEBUG") -else() - set(ARROW_DEFINITION_FLAGS "") -endif() - message(STATUS "Build Type: ${CMAKE_BUILD_TYPE}") # ---------------------------------------------------------------------- diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index 807e2b9206b4..cc37a3c2dfe1 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -29,6 +29,10 @@ add_custom_target(toolchain-tests) # allocators like jemalloc and mimalloc set(ARROW_BUNDLED_STATIC_LIBS) +# Accumulate all system dependencies to provide suitable static link +# parameters to the third party libraries. +set(ARROW_SYSTEM_DEPENDENCIES) + # ---------------------------------------------------------------------- # Toolchain linkage options @@ -66,7 +70,7 @@ set(ARROW_THIRDPARTY_DEPENDENCIES Thrift utf8proc ZLIB - ZSTD) + zstd) # TODO(wesm): External GTest shared libraries are not currently # supported when building with MSVC because of the way that @@ -140,42 +144,68 @@ macro(build_dependency DEPENDENCY_NAME) build_protobuf() elseif("${DEPENDENCY_NAME}" STREQUAL "RE2") build_re2() + elseif("${DEPENDENCY_NAME}" STREQUAL "Snappy") + build_snappy() elseif("${DEPENDENCY_NAME}" STREQUAL "Thrift") build_thrift() elseif("${DEPENDENCY_NAME}" STREQUAL "utf8proc") build_utf8proc() elseif("${DEPENDENCY_NAME}" STREQUAL "ZLIB") build_zlib() - elseif("${DEPENDENCY_NAME}" STREQUAL "ZSTD") + elseif("${DEPENDENCY_NAME}" STREQUAL "zstd") build_zstd() else() message(FATAL_ERROR "Unknown thirdparty dependency to build: ${DEPENDENCY_NAME}") endif() endmacro() -macro(resolve_dependency DEPENDENCY_NAME) - if(${DEPENDENCY_NAME}_SOURCE STREQUAL "AUTO") - find_package(${DEPENDENCY_NAME} MODULE) - if(NOT ${${DEPENDENCY_NAME}_FOUND}) - build_dependency(${DEPENDENCY_NAME}) - endif() - elseif(${DEPENDENCY_NAME}_SOURCE STREQUAL "BUNDLED") - build_dependency(${DEPENDENCY_NAME}) - elseif(${DEPENDENCY_NAME}_SOURCE STREQUAL "SYSTEM") - find_package(${DEPENDENCY_NAME} REQUIRED) +# Find modules are needed by the consumer in case of a static build, or if the +# linkage is PUBLIC or INTERFACE. +macro(provide_find_module DEPENDENCY_NAME) + set(module_ "${CMAKE_SOURCE_DIR}/cmake_modules/Find${DEPENDENCY_NAME}.cmake") + if(EXISTS "${module_}") + message(STATUS "Providing cmake module for ${DEPENDENCY_NAME}") + install(FILES "${module_}" DESTINATION "${ARROW_CMAKE_INSTALL_DIR}") endif() + unset(module_) endmacro() -macro(resolve_dependency_with_version DEPENDENCY_NAME REQUIRED_VERSION) +macro(resolve_dependency DEPENDENCY_NAME) + set(options) + set(one_value_args REQUIRED_VERSION) + cmake_parse_arguments(ARG + "${options}" + "${one_value_args}" + "${multi_value_args}" + ${ARGN}) + if(ARG_UNPARSED_ARGUMENTS) + message(SEND_ERROR "Error: unrecognized arguments: ${ARG_UNPARSED_ARGUMENTS}") + endif() + if(${DEPENDENCY_NAME}_SOURCE STREQUAL "AUTO") - find_package(${DEPENDENCY_NAME} ${REQUIRED_VERSION} MODULE) - if(NOT ${${DEPENDENCY_NAME}_FOUND}) + if(ARG_REQUIRED_VERSION) + find_package(${DEPENDENCY_NAME} ${ARG_REQUIRED_VERSION} MODULE) + else() + find_package(${DEPENDENCY_NAME} MODULE) + endif() + if(${${DEPENDENCY_NAME}_FOUND}) + set(${DEPENDENCY_NAME}_SOURCE "SYSTEM") + else() build_dependency(${DEPENDENCY_NAME}) + set(${DEPENDENCY_NAME}_SOURCE "BUNDLED") endif() elseif(${DEPENDENCY_NAME}_SOURCE STREQUAL "BUNDLED") build_dependency(${DEPENDENCY_NAME}) elseif(${DEPENDENCY_NAME}_SOURCE STREQUAL "SYSTEM") - find_package(${DEPENDENCY_NAME} ${REQUIRED_VERSION} REQUIRED) + if(ARG_REQUIRED_VERSION) + find_package(${DEPENDENCY_NAME} ${ARG_REQUIRED_VERSION} REQUIRED) + else() + find_package(${DEPENDENCY_NAME} REQUIRED) + endif() + endif() + if(${DEPENDENCY_NAME}_SOURCE STREQUAL "SYSTEM") + provide_find_module(${DEPENDENCY_NAME}) + list(APPEND ARROW_SYSTEM_DEPENDENCIES ${DEPENDENCY_NAME}) endif() endmacro() @@ -219,6 +249,10 @@ if(ARROW_ORC OR ARROW_FLIGHT OR ARROW_GANDIVA) set(ARROW_WITH_PROTOBUF ON) endif() +if(ARROW_S3) + set(ARROW_WITH_ZLIB ON) +endif() + if(NOT ARROW_COMPUTE) # utf8proc is only potentially used in kernels for now set(ARROW_WITH_UTF8PROC OFF) @@ -270,6 +304,33 @@ else() "https://github.com/abseil/abseil-cpp/archive/${ARROW_ABSL_BUILD_VERSION}.tar.gz") endif() +if(DEFINED ENV{ARROW_AWS_C_COMMON_URL}) + set(AWS_C_COMMON_SOURCE_URL "$ENV{ARROW_AWS_C_COMMON_URL}") +else() + set_urls( + AWS_C_COMMON_SOURCE_URL + "https://github.com/awslabs/aws-c-common/archive/${ARROW_AWS_C_COMMON_BUILD_VERSION}.tar.gz" + ) +endif() + +if(DEFINED ENV{ARROW_AWS_CHECKSUMS_URL}) + set(AWS_CHECKSUMS_SOURCE_URL "$ENV{ARROW_AWS_CHECKSUMS_URL}") +else() + set_urls( + AWS_CHECKSUMS_SOURCE_URL + "https://github.com/awslabs/aws-checksums/archive/${ARROW_AWS_CHECKSUMS_BUILD_VERSION}.tar.gz" + ) +endif() + +if(DEFINED ENV{ARROW_AWS_C_EVENT_STREAM_URL}) + set(AWS_C_EVENT_STREAM_SOURCE_URL "$ENV{ARROW_AWS_C_EVENT_STREAM_URL}") +else() + set_urls( + AWS_C_EVENT_STREAM_SOURCE_URL + "https://github.com/awslabs/aws-c-event-stream/archive/${ARROW_AWS_C_EVENT_STREAM_BUILD_VERSION}.tar.gz" + ) +endif() + if(DEFINED ENV{ARROW_AWSSDK_URL}) set(AWSSDK_SOURCE_URL "$ENV{ARROW_AWSSDK_URL}") else() @@ -522,7 +583,7 @@ endif() set(EP_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${CMAKE_CXX_FLAGS_${UPPERCASE_BUILD_TYPE}}") set(EP_C_FLAGS "${CMAKE_C_FLAGS} ${CMAKE_C_FLAGS_${UPPERCASE_BUILD_TYPE}}") -if(NOT MSVC) +if(NOT MSVC_TOOLCHAIN) # Set -fPIC on all external projects set(EP_CXX_FLAGS "${EP_CXX_FLAGS} -fPIC") set(EP_C_FLAGS "${EP_C_FLAGS} -fPIC") @@ -847,31 +908,7 @@ macro(build_snappy) endmacro() if(ARROW_WITH_SNAPPY) - if(Snappy_SOURCE STREQUAL "AUTO") - # Normally *Config.cmake files reside in /usr/lib/cmake but Snappy - # errornously places them in ${CMAKE_ROOT}/Modules/ - # This is fixed in 1.1.7 but fedora (30) still installs into the wrong - # location. - # https://bugzilla.redhat.com/show_bug.cgi?id=1679727 - # https://src.fedoraproject.org/rpms/snappy/pull-request/1 - find_package(Snappy QUIET HINTS "${CMAKE_ROOT}/Modules/") - if(NOT Snappy_FOUND) - find_package(SnappyAlt) - endif() - if(NOT Snappy_FOUND AND NOT SnappyAlt_FOUND) - build_snappy() - endif() - elseif(Snappy_SOURCE STREQUAL "BUNDLED") - build_snappy() - elseif(Snappy_SOURCE STREQUAL "SYSTEM") - # SnappyConfig.cmake is not installed on Ubuntu/Debian - # TODO: Make a bug report upstream - find_package(Snappy HINTS "${CMAKE_ROOT}/Modules/") - if(NOT Snappy_FOUND) - find_package(SnappyAlt REQUIRED) - endif() - endif() - + resolve_dependency(Snappy) # TODO: Don't use global includes but rather target_include_directories get_target_property(SNAPPY_INCLUDE_DIRS Snappy::snappy INTERFACE_INCLUDE_DIRECTORIES) include_directories(SYSTEM ${SNAPPY_INCLUDE_DIRS}) @@ -965,8 +1002,24 @@ endif() set(ARROW_USE_OPENSSL OFF) if(PARQUET_REQUIRE_ENCRYPTION OR ARROW_FLIGHT OR ARROW_S3) - # This must work - find_package(OpenSSL ${ARROW_OPENSSL_REQUIRED_VERSION} REQUIRED) + # OpenSSL is required + if(ARROW_OPENSSL_USE_SHARED) + # Find shared OpenSSL libraries. + set(OpenSSL_USE_STATIC_LIBS OFF) + # Seems that different envs capitalize this differently? + set(OPENSSL_USE_STATIC_LIBS OFF) + set(BUILD_SHARED_LIBS_KEEP ${BUILD_SHARED_LIBS}) + set(BUILD_SHARED_LIBS ON) + + find_package(OpenSSL ${ARROW_OPENSSL_REQUIRED_VERSION} REQUIRED) + set(BUILD_SHARED_LIBS ${BUILD_SHARED_LIBS_KEEP}) + unset(BUILD_SHARED_LIBS_KEEP) + else() + # Find static OpenSSL headers and libs + set(OpenSSL_USE_STATIC_LIBS ON) + set(OPENSSL_USE_STATIC_LIBS ON) + find_package(OpenSSL ${ARROW_OPENSSL_REQUIRED_VERSION} REQUIRED) + endif() set(ARROW_USE_OPENSSL ON) endif() @@ -990,6 +1043,7 @@ if(ARROW_USE_OPENSSL) INTERFACE_INCLUDE_DIRECTORIES "${OPENSSL_INCLUDE_DIR}") endif() + list(APPEND ARROW_SYSTEM_DEPENDENCIES "OpenSSL") include_directories(SYSTEM ${OPENSSL_INCLUDE_DIR}) else() @@ -1239,7 +1293,7 @@ if(ARROW_WITH_THRIFT) # to build Boost, so don't look again if already found. if(NOT Thrift_FOUND AND NOT THRIFT_FOUND) # Thrift c++ code generated by 0.13 requires 0.11 or greater - resolve_dependency_with_version(Thrift 0.11.0) + resolve_dependency(Thrift REQUIRED_VERSION 0.11.0) endif() # TODO: Don't use global includes but rather target_include_directories include_directories(SYSTEM ${THRIFT_INCLUDE_DIR}) @@ -1339,9 +1393,9 @@ if(ARROW_WITH_PROTOBUF) else() set(ARROW_PROTOBUF_REQUIRED_VERSION "2.6.1") endif() - resolve_dependency_with_version(Protobuf ${ARROW_PROTOBUF_REQUIRED_VERSION}) + resolve_dependency(Protobuf REQUIRED_VERSION ${ARROW_PROTOBUF_REQUIRED_VERSION}) - if(ARROW_PROTOBUF_USE_SHARED AND MSVC) + if(ARROW_PROTOBUF_USE_SHARED AND MSVC_TOOLCHAIN) add_definitions(-DPROTOBUF_USE_DLLS) endif() @@ -1447,7 +1501,7 @@ if(ARROW_JEMALLOC) BUILD_IN_SOURCE 1 BUILD_COMMAND ${JEMALLOC_BUILD_COMMAND} BUILD_BYPRODUCTS "${JEMALLOC_STATIC_LIB}" - INSTALL_COMMAND ${MAKE} install) + INSTALL_COMMAND ${MAKE} -j1 install) # Don't use the include directory directly so that we can point to a path # that is unique to our codebase. @@ -1640,10 +1694,7 @@ macro(build_gtest) add_dependencies(GTest::gmock googletest_ep) endmacro() -if(ARROW_BUILD_TESTS - OR ARROW_BUILD_BENCHMARKS - OR ARROW_BUILD_INTEGRATION - OR ARROW_FUZZING) +if(ARROW_TESTING) resolve_dependency(GTest) if(NOT GTEST_VENDORED) @@ -1843,9 +1894,11 @@ macro(build_zlib) file(MAKE_DIRECTORY "${ZLIB_PREFIX}/include") add_library(ZLIB::ZLIB STATIC IMPORTED) + set(ZLIB_LIBRARIES ${ZLIB_STATIC_LIB}) + set(ZLIB_INCLUDE_DIRS "${ZLIB_PREFIX}/include") set_target_properties(ZLIB::ZLIB - PROPERTIES IMPORTED_LOCATION "${ZLIB_STATIC_LIB}" - INTERFACE_INCLUDE_DIRECTORIES "${ZLIB_PREFIX}/include") + PROPERTIES IMPORTED_LOCATION ${ZLIB_LIBRARIES} + INTERFACE_INCLUDE_DIRECTORIES ${ZLIB_INCLUDE_DIRS}) add_dependencies(toolchain zlib_ep) add_dependencies(ZLIB::ZLIB zlib_ep) @@ -1982,7 +2035,7 @@ macro(build_zstd) endmacro() if(ARROW_WITH_ZSTD) - resolve_dependency(ZSTD) + resolve_dependency(zstd) if(TARGET zstd::libzstd) set(ARROW_ZSTD_LIBZSTD zstd::libzstd) @@ -2186,6 +2239,14 @@ macro(build_cares) set_target_properties(c-ares::cares PROPERTIES IMPORTED_LOCATION "${CARES_STATIC_LIB}" INTERFACE_INCLUDE_DIRECTORIES "${CARES_INCLUDE_DIR}") + add_dependencies(c-ares::cares cares_ep) + + if(APPLE) + # libresolv must be linked from c-ares version 1.16.1 + find_library(LIBRESOLV_LIBRARY NAMES resolv libresolv REQUIRED) + set_target_properties(c-ares::cares + PROPERTIES INTERFACE_LINK_LIBRARIES "${LIBRESOLV_LIBRARY}") + endif() set(CARES_VENDORED TRUE) @@ -2604,12 +2665,14 @@ endif() # AWS SDK for C++ macro(build_awssdk) - message( - FATAL_ERROR "FIXME: Building AWS C++ SDK from source will link with wrong libcrypto") message("Building AWS C++ SDK from source") - + if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU" + AND CMAKE_CXX_COMPILER_VERSION VERSION_LESS "4.9") + message(FATAL_ERROR "AWS C++ SDK requires gcc >= 4.9") + endif() set(AWSSDK_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/awssdk_ep-install") set(AWSSDK_INCLUDE_DIR "${AWSSDK_PREFIX}/include") + set(AWSSDK_LIB_DIR "lib") if(WIN32) # On Windows, need to match build types @@ -2618,56 +2681,178 @@ macro(build_awssdk) # Otherwise, always build in release mode. # Especially with gcc, debug builds can fail with "asm constraint" errors: # https://github.com/TileDB-Inc/TileDB/issues/1351 - set(AWSSDK_BUILD_TYPE Release) + set(AWSSDK_BUILD_TYPE release) endif() - set(AWSSDK_CMAKE_ARGS - -DCMAKE_BUILD_TYPE=Release - -DCMAKE_INSTALL_LIBDIR=lib - -DBUILD_ONLY=s3;core;config - -DENABLE_UNITY_BUILD=on - -DENABLE_TESTING=off - "-DCMAKE_C_FLAGS=${EP_C_FLAGS}" - "-DCMAKE_INSTALL_PREFIX=${AWSSDK_PREFIX}") + set(AWSSDK_COMMON_CMAKE_ARGS + ${EP_COMMON_CMAKE_ARGS} + -DBUILD_SHARED_LIBS=OFF + -DCMAKE_BUILD_TYPE=${AWSSDK_BUILD_TYPE} + -DCMAKE_INSTALL_LIBDIR=${AWSSDK_LIB_DIR} + -DENABLE_TESTING=OFF + -DENABLE_UNITY_BUILD=ON + "-DCMAKE_INSTALL_PREFIX=${AWSSDK_PREFIX}" + "-DCMAKE_PREFIX_PATH=${AWSSDK_PREFIX}") set( - AWSSDK_CORE_SHARED_LIB - "${AWSSDK_PREFIX}/lib/${CMAKE_SHARED_LIBRARY_PREFIX}aws-cpp-sdk-core${CMAKE_SHARED_LIBRARY_SUFFIX}" - ) - set( - AWSSDK_S3_SHARED_LIB - "${AWSSDK_PREFIX}/lib/${CMAKE_SHARED_LIBRARY_PREFIX}aws-cpp-sdk-s3${CMAKE_SHARED_LIBRARY_SUFFIX}" - ) - set(AWSSDK_SHARED_LIBS "${AWSSDK_CORE_SHARED_LIB}" "${AWSSDK_S3_SHARED_LIB}") + AWSSDK_CMAKE_ARGS + ${AWSSDK_COMMON_CMAKE_ARGS} -DBUILD_DEPS=OFF + -DBUILD_ONLY=config\\$s3\\$transfer\\$identity-management\\$sts + -DMINIMIZE_SIZE=ON) + if(UNIX AND TARGET zlib_ep) + list(APPEND AWSSDK_CMAKE_ARGS -DZLIB_INCLUDE_DIR=${ZLIB_INCLUDE_DIRS} + -DZLIB_LIBRARY=${ZLIB_LIBRARIES}) + endif() + + file(MAKE_DIRECTORY ${AWSSDK_INCLUDE_DIR}) + + # AWS C++ SDK related libraries to link statically + set(_AWSSDK_LIBS + aws-cpp-sdk-identity-management + aws-cpp-sdk-sts + aws-cpp-sdk-cognito-identity + aws-cpp-sdk-s3 + aws-cpp-sdk-core + aws-c-event-stream + aws-checksums + aws-c-common) + set(AWSSDK_LIBRARIES) + foreach(_AWSSDK_LIB ${_AWSSDK_LIBS}) + # aws-c-common -> AWS-C-COMMON + string(TOUPPER ${_AWSSDK_LIB} _AWSSDK_LIB_UPPER) + # AWS-C-COMMON -> AWS_C_COMMON + string(REPLACE "-" "_" _AWSSDK_LIB_NAME_PREFIX ${_AWSSDK_LIB_UPPER}) + set( + _AWSSDK_STATIC_LIBRARY + "${AWSSDK_PREFIX}/${AWSSDK_LIB_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}${_AWSSDK_LIB}${CMAKE_STATIC_LIBRARY_SUFFIX}" + ) + if(${_AWSSDK_LIB} MATCHES "^aws-cpp-sdk-") + set(_AWSSDK_TARGET_NAME ${_AWSSDK_LIB}) + else() + set(_AWSSDK_TARGET_NAME AWS::${_AWSSDK_LIB}) + endif() + add_library(${_AWSSDK_TARGET_NAME} STATIC IMPORTED) + set_target_properties( + ${_AWSSDK_TARGET_NAME} + PROPERTIES IMPORTED_LOCATION ${_AWSSDK_STATIC_LIBRARY} INTERFACE_INCLUDE_DIRECTORIES + "${AWSSDK_INCLUDE_DIR}") + set("${_AWSSDK_LIB_NAME_PREFIX}_STATIC_LIBRARY" ${_AWSSDK_STATIC_LIBRARY}) + list(APPEND AWSSDK_LIBRARIES ${_AWSSDK_TARGET_NAME}) + endforeach() + + externalproject_add(aws_c_common_ep + ${EP_LOG_OPTIONS} + URL ${AWS_C_COMMON_SOURCE_URL} + CMAKE_ARGS ${AWSSDK_COMMON_CMAKE_ARGS} + BUILD_BYPRODUCTS ${AWS_C_COMMON_STATIC_LIBRARY}) + add_dependencies(AWS::aws-c-common aws_c_common_ep) + + externalproject_add(aws_checksums_ep + ${EP_LOG_OPTIONS} + URL ${AWS_CHECKSUMS_SOURCE_URL} + CMAKE_ARGS ${AWSSDK_COMMON_CMAKE_ARGS} + BUILD_BYPRODUCTS ${AWS_CHECKSUMS_STATIC_LIBRARY}) + add_dependencies(AWS::aws-checksums aws_checksums_ep) + + externalproject_add(aws_c_event_stream_ep + ${EP_LOG_OPTIONS} + URL ${AWS_C_EVENT_STREAM_SOURCE_URL} + CMAKE_ARGS ${AWSSDK_COMMON_CMAKE_ARGS} + BUILD_BYPRODUCTS ${AWS_C_EVENT_STREAM_STATIC_LIBRARY} + DEPENDS aws_c_common_ep aws_checksums_ep) + add_dependencies(AWS::aws-c-event-stream aws_c_event_stream_ep) externalproject_add(awssdk_ep ${EP_LOG_OPTIONS} URL ${AWSSDK_SOURCE_URL} CMAKE_ARGS ${AWSSDK_CMAKE_ARGS} - BUILD_BYPRODUCTS ${AWSSDK_SHARED_LIBS}) - - file(MAKE_DIRECTORY ${AWSSDK_INCLUDE_DIR}) - + BUILD_BYPRODUCTS ${AWS_CPP_SDK_COGNITO_IDENTITY_STATIC_LIBRARY} + ${AWS_CPP_SDK_CORE_STATIC_LIBRARY} + ${AWS_CPP_SDK_IDENTITY_MANAGEMENT_STATIC_LIBRARY} + ${AWS_CPP_SDK_S3_STATIC_LIBRARY} + ${AWS_CPP_SDK_STS_STATIC_LIBRARY} + DEPENDS aws_c_event_stream_ep) add_dependencies(toolchain awssdk_ep) - set(AWSSDK_LINK_LIBRARIES ${AWSSDK_SHARED_LIBS}) + foreach(_AWSSDK_LIB ${_AWSSDK_LIBS}) + if(${_AWSSDK_LIB} MATCHES "^aws-cpp-sdk-") + add_dependencies(${_AWSSDK_LIB} awssdk_ep) + endif() + endforeach() + set(AWSSDK_VENDORED TRUE) + list(APPEND ARROW_BUNDLED_STATIC_LIBS ${AWSSDK_LIBRARIES}) + set(AWSSDK_LINK_LIBRARIES ${AWSSDK_LIBRARIES}) + if(UNIX) + # on linux and macos curl seems to be required + find_package(CURL REQUIRED) + if(NOT TARGET CURL::libcurl) + # For old FindCURL.cmake + add_library(CURL::libcurl UNKNOWN IMPORTED) + set_target_properties(CURL::libcurl + PROPERTIES INTERFACE_INCLUDE_DIRECTORIES + "${CURL_INCLUDE_DIRS}" IMPORTED_LOCATION + "${CURL_LIBRARIES}") + endif() + set_property(TARGET aws-cpp-sdk-core + APPEND + PROPERTY INTERFACE_LINK_LIBRARIES CURL::libcurl) + set_property(TARGET CURL::libcurl + APPEND + PROPERTY INTERFACE_LINK_LIBRARIES OpenSSL::SSL) + if(TARGET zlib_ep) + set_property(TARGET aws-cpp-sdk-core + APPEND + PROPERTY INTERFACE_LINK_LIBRARIES ZLIB::ZLIB) + add_dependencies(awssdk_ep zlib_ep) + endif() + endif() - # AWSSDK is shared-only build + # AWSSDK is static-only build endmacro() if(ARROW_S3) # See https://aws.amazon.com/blogs/developer/developer-experience-of-the-aws-sdk-for-c-now-simplified-by-cmake/ + # Workaround to force AWS cmake configuration to look for shared libraries + if(DEFINED ENV{CONDA_PREFIX}) + if(DEFINED BUILD_SHARED_LIBS) + set(BUILD_SHARED_LIBS_WAS_SET TRUE) + set(BUILD_SHARED_LIBS_VALUE ${BUILD_SHARED_LIBS}) + else() + set(BUILD_SHARED_LIBS_WAS_SET FALSE) + endif() + set(BUILD_SHARED_LIBS "ON") + endif() + # Need to customize the find_package() call, so cannot call resolve_dependency() if(AWSSDK_SOURCE STREQUAL "AUTO") - find_package(AWSSDK COMPONENTS config s3 transfer) + find_package(AWSSDK + COMPONENTS config + s3 + transfer + identity-management + sts) if(NOT AWSSDK_FOUND) build_awssdk() endif() elseif(AWSSDK_SOURCE STREQUAL "BUNDLED") build_awssdk() elseif(AWSSDK_SOURCE STREQUAL "SYSTEM") - find_package(AWSSDK REQUIRED COMPONENTS config s3 transfer) + find_package(AWSSDK REQUIRED + COMPONENTS config + s3 + transfer + identity-management + sts) + endif() + + # Restore previous value of BUILD_SHARED_LIBS + if(DEFINED ENV{CONDA_PREFIX}) + if(BUILD_SHARED_LIBS_WAS_SET) + set(BUILD_SHARED_LIBS ${BUILD_SHARED_LIBS_VALUE}) + else() + unset(BUILD_SHARED_LIBS) + endif() endif() include_directories(SYSTEM ${AWSSDK_INCLUDE_DIR}) @@ -2689,6 +2874,6 @@ message(STATUS "All bundled static libraries: ${ARROW_BUNDLED_STATIC_LIBS}") # Write out the package configurations. -configure_file("src/arrow/util/config.h.cmake" "src/arrow/util/config.h") +configure_file("src/arrow/util/config.h.cmake" "src/arrow/util/config.h" ESCAPE_QUOTES) install(FILES "${ARROW_BINARY_DIR}/src/arrow/util/config.h" DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/arrow/util") diff --git a/cpp/cmake_modules/san-config.cmake b/cpp/cmake_modules/san-config.cmake index 2e2807801fb9..5eee6278009e 100644 --- a/cpp/cmake_modules/san-config.cmake +++ b/cpp/cmake_modules/san-config.cmake @@ -35,14 +35,17 @@ endif() # - disable 'vptr' because of RTTI issues across shared libraries (?) # - disable 'alignment' because unaligned access is really OK on Nehalem and we do it # all over the place. -# - disable 'function' because it appears to give a false positive https://github.com/google/sanitizers/issues/911 +# - disable 'function' because it appears to give a false positive +# (https://github.com/google/sanitizers/issues/911) +# - disable 'float-divide-by-zero' on clang, which considers it UB +# (https://bugs.llvm.org/show_bug.cgi?id=17000#c1) # Note: GCC does not support the 'function' flag. if(${ARROW_USE_UBSAN}) if(CMAKE_CXX_COMPILER_ID STREQUAL "AppleClang" OR CMAKE_CXX_COMPILER_ID STREQUAL "Clang") set( CMAKE_CXX_FLAGS - "${CMAKE_CXX_FLAGS} -fsanitize=undefined -fno-sanitize=alignment,vptr,function -fno-sanitize-recover=all" + "${CMAKE_CXX_FLAGS} -fsanitize=undefined -fno-sanitize=alignment,vptr,function,float-divide-by-zero -fno-sanitize-recover=all" ) elseif(CMAKE_CXX_COMPILER_ID STREQUAL "GNU" AND CMAKE_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL "5.1") diff --git a/cpp/examples/minimal_build/README.md b/cpp/examples/minimal_build/README.md index b79d20c2a0e2..9f889f6adda1 100644 --- a/cpp/examples/minimal_build/README.md +++ b/cpp/examples/minimal_build/README.md @@ -30,20 +30,14 @@ a file named `test.arrow`. ## Running the example -You can run this simple example using `Docker` and the given `Dockerfile`, -which installs a minimal Ubuntu image with a basic C++ toolchain. +You can run this simple example using [Docker Compose][docker-compose] +and the given `docker-compose.yml` and dockerfiles, which installs a +minimal Ubuntu image with a basic C++ toolchain. -Just open a terminal in this directory and, assuming `$ARROW_ROOT` points to -the Arrow C++ source tree (either a stable release or a git checkout), run -the following commands: +Just open a terminal in this directory and run the following commands: ```bash - -# Build the Docker image -docker build -t arrow_cpp_minimal . - -# Run the example inside the image -docker run -it -v $PWD:/io -v $ARROW_ROOT:/arrow arrow_cpp_minimal /io/run.sh +docker-compose run --rm minimal ``` Note that this example mounts two volumes inside the Docker image: @@ -53,12 +47,12 @@ Note that this example mounts two volumes inside the Docker image: ## Statically-linked builds We've provided an example build configuration here with CMake to show how to -create a statically-linked executable . +create a statically-linked executable with bundled dependencies. To run it on Linux, you can use the above Docker image: ```bash -docker run -it -v $PWD:/io -v $ARROW_ROOT:/arrow arrow_cpp_minimal /io/run_static.sh +docker-compose run --rm static ``` On macOS, you can use the `run_static.sh` but you must set some environment @@ -67,8 +61,8 @@ variables to point the script to your Arrow checkout, for example: ```bash export ARROW_DIR=path/to/arrow-clone export EXAMPLE_DIR=$ARROW_DIR/cpp/examples/minimal_build -export ARROW_BUILD_DIR=`pwd`/arrow-build -export EXAMPLE_BUILD_DIR=`pwd`/example +export ARROW_BUILD_DIR=$(pwd)/arrow-build +export EXAMPLE_BUILD_DIR=$(pwd)/example ./run_static.sh ``` @@ -78,4 +72,17 @@ Studio's command line tools enabled and CMake and ninja build in the path: ``` call run_static.bat -``` \ No newline at end of file +``` + +### Static linking against system libraries + +You can also use static libraries of Arrow's dependencies from the +system. To run this configuration, set +`ARROW_DEPENDENCY_SOURCE=SYSTEM` for `run_static.sh`. You can use +`docker-compose` for this too: + +```bash +docker-compose run --rm static-system-dependency +``` + +[docker-compose]: https://docs.docker.com/compose/ diff --git a/cpp/examples/minimal_build/docker-compose.yml b/cpp/examples/minimal_build/docker-compose.yml new file mode 100644 index 000000000000..6e2dcef81213 --- /dev/null +++ b/cpp/examples/minimal_build/docker-compose.yml @@ -0,0 +1,51 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +version: '3.5' + +services: + minimal: + build: + context: . + dockerfile: minimal.dockerfile + volumes: + - ../../../:/arrow:delegated + - .:/io:delegated + command: + - "/io/run.sh" + + static: + build: + context: . + dockerfile: minimal.dockerfile + volumes: + - ../../../:/arrow:delegated + - .:/io:delegated + command: + - "/io/run_static.sh" + + static-system-dependency: + build: + context: . + dockerfile: system_dependency.dockerfile + environment: + ARROW_DEPENDENCY_SOURCE: "SYSTEM" + volumes: + - ../../../:/arrow:delegated + - .:/io:delegated + command: + - "/io/run_static.sh" diff --git a/cpp/examples/minimal_build/example.cc b/cpp/examples/minimal_build/example.cc index 7c60093e95fb..4b6acd2a0dd7 100644 --- a/cpp/examples/minimal_build/example.cc +++ b/cpp/examples/minimal_build/example.cc @@ -52,8 +52,8 @@ Status RunMain(int argc, char** argv) { ARROW_ASSIGN_OR_RAISE(auto output_file, arrow::io::FileOutputStream::Open(arrow_filename)); ARROW_ASSIGN_OR_RAISE(auto batch_writer, - arrow::ipc::NewFileWriter(output_file.get(), - table->schema())); + arrow::ipc::MakeFileWriter(output_file, + table->schema())); ARROW_RETURN_NOT_OK(batch_writer->WriteTable(*table)); ARROW_RETURN_NOT_OK(batch_writer->Close()); diff --git a/cpp/examples/minimal_build/Dockerfile b/cpp/examples/minimal_build/minimal.dockerfile similarity index 93% rename from cpp/examples/minimal_build/Dockerfile rename to cpp/examples/minimal_build/minimal.dockerfile index 6f00f475bdc3..95f73e9a5490 100644 --- a/cpp/examples/minimal_build/Dockerfile +++ b/cpp/examples/minimal_build/minimal.dockerfile @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. -FROM ubuntu:bionic +FROM ubuntu:focal ENV DEBIAN_FRONTEND=noninteractive @@ -23,4 +23,4 @@ RUN apt-get update -y -q && \ apt-get install -y -q --no-install-recommends \ build-essential \ cmake && \ - apt-get clean && rm -rf /var/lib/apt/lists* + apt-get clean && rm -rf /var/lib/apt/lists* diff --git a/cpp/examples/minimal_build/run_static.sh b/cpp/examples/minimal_build/run_static.sh index 941a29f31866..05804a0366cc 100755 --- a/cpp/examples/minimal_build/run_static.sh +++ b/cpp/examples/minimal_build/run_static.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information @@ -23,6 +23,8 @@ set -e : ${ARROW_BUILD_DIR:=/build/arrow} : ${EXAMPLE_BUILD_DIR:=/build/example} +: ${ARROW_DEPENDENCY_SOURCE:=BUNDLED} + echo echo "==" echo "== Building Arrow C++ library" @@ -35,12 +37,13 @@ pushd $ARROW_BUILD_DIR NPROC=$(nproc) cmake $ARROW_DIR/cpp \ - -DARROW_DEPENDENCY_SOURCE=BUNDLED \ -DARROW_BUILD_SHARED=OFF \ -DARROW_BUILD_STATIC=ON \ -DARROW_COMPUTE=ON \ -DARROW_CSV=ON \ -DARROW_DATASET=ON \ + -DARROW_DEPENDENCY_SOURCE=${ARROW_DEPENDENCY_SOURCE} \ + -DARROW_DEPENDENCY_USE_SHARED=OFF \ -DARROW_FILESYSTEM=ON \ -DARROW_HDFS=ON \ -DARROW_JEMALLOC=ON \ @@ -54,6 +57,7 @@ cmake $ARROW_DIR/cpp \ -DARROW_WITH_SNAPPY=ON \ -DARROW_WITH_ZLIB=ON \ -DARROW_WITH_ZSTD=ON \ + -DORC_SOURCE=BUNDLED \ $ARROW_CMAKE_OPTIONS make -j$NPROC @@ -81,6 +85,6 @@ echo "== Running example project" echo "==" echo -cd $EXAMPLE_DIR +pushd $EXAMPLE_DIR ${EXAMPLE_BUILD_DIR}/arrow_example diff --git a/cpp/examples/minimal_build/system_dependency.dockerfile b/cpp/examples/minimal_build/system_dependency.dockerfile new file mode 100644 index 000000000000..6615530bee15 --- /dev/null +++ b/cpp/examples/minimal_build/system_dependency.dockerfile @@ -0,0 +1,42 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +FROM ubuntu:focal + +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt-get update -y -q && \ + apt-get install -y -q --no-install-recommends \ + build-essential \ + cmake \ + libboost-filesystem-dev \ + libboost-regex-dev \ + libboost-system-dev \ + libbrotli-dev \ + libbz2-dev \ + libgflags-dev \ + liblz4-dev \ + libprotobuf-dev \ + libprotoc-dev \ + libsnappy-dev \ + libthrift-dev \ + libutf8proc-dev \ + libzstd-dev \ + protobuf-compiler \ + rapidjson-dev \ + zlib1g-dev && \ + apt-get clean && rm -rf /var/lib/apt/lists* diff --git a/cpp/examples/parquet/parquet-stream-api/stream-reader-writer.cc b/cpp/examples/parquet/parquet-stream-api/stream-reader-writer.cc index 5a255bff2759..64e44803ccfb 100644 --- a/cpp/examples/parquet/parquet-stream-api/stream-reader-writer.cc +++ b/cpp/examples/parquet/parquet-stream-api/stream-reader-writer.cc @@ -48,7 +48,8 @@ class UserTimestamp { bool operator==(const UserTimestamp& x) const { return ts_ == x.ts_; } void dump(std::ostream& os) const { - std::time_t t{std::chrono::duration_cast(ts_).count()}; + const auto t = static_cast( + std::chrono::duration_cast(ts_).count()); os << std::put_time(std::gmtime(&t), "%Y%m%d-%H%M%S"); } diff --git a/cpp/src/arrow/ArrowConfig.cmake.in b/cpp/src/arrow/ArrowConfig.cmake.in index c4b35d5aba36..0cf8a85f570a 100644 --- a/cpp/src/arrow/ArrowConfig.cmake.in +++ b/cpp/src/arrow/ArrowConfig.cmake.in @@ -34,14 +34,40 @@ set(ARROW_VERSION "@ARROW_VERSION@") set(ARROW_SO_VERSION "@ARROW_SO_VERSION@") set(ARROW_FULL_SO_VERSION "@ARROW_FULL_SO_VERSION@") +set(ARROW_LIBRARY_PATH_SUFFIXES "@ARROW_LIBRARY_PATH_SUFFIXES@") +set(ARROW_INCLUDE_PATH_SUFFIXES "@ARROW_INCLUDE_PATH_SUFFIXES@") +set(ARROW_SYSTEM_DEPENDENCIES "@ARROW_SYSTEM_DEPENDENCIES@") + include("${CMAKE_CURRENT_LIST_DIR}/ArrowOptions.cmake") +include(CMakeFindDependencyMacro) + # Load targets only once. If we load targets multiple times, CMake reports # already existent target error. if(NOT (TARGET arrow_shared OR TARGET arrow_static)) include("${CMAKE_CURRENT_LIST_DIR}/ArrowTargets.cmake") if(TARGET arrow_static) + set(CMAKE_THREAD_PREFER_PTHREAD TRUE) + set(THREADS_PREFER_PTHREAD_FLAG TRUE) + find_dependency(Threads) + + if(DEFINED CMAKE_MODULE_PATH) + set(_CMAKE_MODULE_PATH_OLD ${CMAKE_MODULE_PATH}) + endif() + set(CMAKE_MODULE_PATH "${CMAKE_CURRENT_LIST_DIR}") + + foreach(_DEPENDENCY ${ARROW_SYSTEM_DEPENDENCIES}) + find_dependency(${_DEPENDENCY}) + endforeach() + + if(DEFINED _CMAKE_MODULE_PATH_OLD) + set(CMAKE_MODULE_PATH ${_CMAKE_MODULE_PATH_OLD}) + unset(_CMAKE_MODULE_PATH_OLD) + else() + unset(CMAKE_MODULE_PATH) + endif() + get_property(arrow_static_loc TARGET arrow_static PROPERTY LOCATION) get_filename_component(arrow_lib_dir ${arrow_static_loc} DIRECTORY) diff --git a/cpp/src/arrow/CMakeLists.txt b/cpp/src/arrow/CMakeLists.txt index 91e67fb423a2..bbeed8df2925 100644 --- a/cpp/src/arrow/CMakeLists.txt +++ b/cpp/src/arrow/CMakeLists.txt @@ -180,6 +180,7 @@ set(ARROW_SRCS util/bitmap.cc util/bitmap_builders.cc util/bitmap_ops.cc + util/bpacking.cc util/compression.cc util/cpu_info.cc util/decimal.cc @@ -213,6 +214,20 @@ set(ARROW_SRCS vendored/double-conversion/diy-fp.cc vendored/double-conversion/strtod.cc) +if(CXX_SUPPORTS_AVX2) + list(APPEND ARROW_SRCS util/bpacking_avx2.cc) + set_source_files_properties(util/bpacking_avx2.cc PROPERTIES SKIP_PRECOMPILE_HEADERS ON) + set_source_files_properties(util/bpacking_avx2.cc PROPERTIES COMPILE_FLAGS + ${ARROW_AVX2_FLAG}) +endif() +if(CXX_SUPPORTS_AVX512) + list(APPEND ARROW_SRCS util/bpacking_avx512.cc) + set_source_files_properties(util/bpacking_avx512.cc PROPERTIES SKIP_PRECOMPILE_HEADERS + ON) + set_source_files_properties(util/bpacking_avx512.cc PROPERTIES COMPILE_FLAGS + ${ARROW_AVX512_FLAG}) +endif() + if(APPLE) list(APPEND ARROW_SRCS vendored/datetime/ios.mm) endif() @@ -349,6 +364,8 @@ if(ARROW_COMPUTE) compute/kernel.cc compute/registry.cc compute/kernels/aggregate_basic.cc + compute/kernels/aggregate_mode.cc + compute/kernels/aggregate_var_std.cc compute/kernels/codegen_internal.cc compute/kernels/scalar_arithmetic.cc compute/kernels/scalar_boolean.cc @@ -371,17 +388,17 @@ if(ARROW_COMPUTE) compute/kernels/vector_sort.cc) if(CXX_SUPPORTS_AVX2) - list(APPEND ARROW_SRCS compute/kernels/aggregate_sum_avx2.cc) - set_source_files_properties(compute/kernels/aggregate_sum_avx2.cc PROPERTIES + list(APPEND ARROW_SRCS compute/kernels/aggregate_basic_avx2.cc) + set_source_files_properties(compute/kernels/aggregate_basic_avx2.cc PROPERTIES SKIP_PRECOMPILE_HEADERS ON) - set_source_files_properties(compute/kernels/aggregate_sum_avx2.cc PROPERTIES + set_source_files_properties(compute/kernels/aggregate_basic_avx2.cc PROPERTIES COMPILE_FLAGS ${ARROW_AVX2_FLAG}) endif() if(CXX_SUPPORTS_AVX512) - list(APPEND ARROW_SRCS compute/kernels/aggregate_sum_avx512.cc) - set_source_files_properties(compute/kernels/aggregate_sum_avx512.cc PROPERTIES + list(APPEND ARROW_SRCS compute/kernels/aggregate_basic_avx512.cc) + set_source_files_properties(compute/kernels/aggregate_basic_avx512.cc PROPERTIES SKIP_PRECOMPILE_HEADERS ON) - set_source_files_properties(compute/kernels/aggregate_sum_avx512.cc PROPERTIES + set_source_files_properties(compute/kernels/aggregate_basic_avx512.cc PROPERTIES COMPILE_FLAGS ${ARROW_AVX512_FLAG}) endif() endif() @@ -390,9 +407,6 @@ if(ARROW_FILESYSTEM) if(ARROW_HDFS) add_definitions(-DARROW_HDFS) endif() - if(ARROW_S3) - add_definitions(-DARROW_S3) - endif() list(APPEND ARROW_SRCS filesystem/filesystem.cc @@ -447,7 +461,7 @@ if(ARROW_ORC) list(APPEND ARROW_SRCS adapters/orc/adapter.cc adapters/orc/adapter_util.cc) endif() -if(NOT APPLE AND NOT MSVC) +if(NOT APPLE AND NOT MSVC_TOOLCHAIN) # Localize thirdparty symbols using a linker version script. This hides them # from the client application. The OS X linker does not support the # version-script option. @@ -513,11 +527,7 @@ if(ARROW_BUILD_STATIC AND ARROW_BUNDLED_STATIC_LIBS) ${_OTHER_LIBS}) endif() -if(ARROW_BUILD_TESTS - OR ARROW_BUILD_BENCHMARKS - OR ARROW_BUILD_INTEGRATION - OR ARROW_FUZZING) - +if(ARROW_TESTING) # that depend on gtest add_arrow_lib(arrow_testing CMAKE_PACKAGE_NAME diff --git a/cpp/src/arrow/array/array_base.cc b/cpp/src/arrow/array/array_base.cc index 0781dd4a2df6..900e8d2b38f1 100644 --- a/cpp/src/arrow/array/array_base.cc +++ b/cpp/src/arrow/array/array_base.cc @@ -161,7 +161,13 @@ struct ScalarFromArraySlotImpl { } if (array_.IsNull(index_)) { - return MakeNullScalar(array_.type()); + auto null = MakeNullScalar(array_.type()); + if (is_dictionary(array_.type()->id())) { + auto& dict_null = checked_cast(*null); + const auto& dict_array = checked_cast(array_); + dict_null.value.dictionary = dict_array.dictionary(); + } + return null; } RETURN_NOT_OK(VisitArrayInline(array_, this)); diff --git a/cpp/src/arrow/array/array_base.h b/cpp/src/arrow/array/array_base.h index 808889be73f1..ef0b1b626e33 100644 --- a/cpp/src/arrow/array/array_base.h +++ b/cpp/src/arrow/array/array_base.h @@ -187,9 +187,9 @@ class ARROW_EXPORT Array { const uint8_t* null_bitmap_data_; /// Protected method for constructors - inline void SetData(const std::shared_ptr& data) { - if (data->buffers.size() > 0 && data->buffers[0]) { - null_bitmap_data_ = data->buffers[0]->data(); + void SetData(const std::shared_ptr& data) { + if (data->buffers.size() > 0) { + null_bitmap_data_ = data->GetValuesSafe(0, /*offset=*/0); } else { null_bitmap_data_ = NULLPTR; } @@ -225,15 +225,12 @@ class ARROW_EXPORT PrimitiveArray : public FlatArray { protected: PrimitiveArray() : raw_values_(NULLPTR) {} - inline void SetData(const std::shared_ptr& data) { - auto values = data->buffers[1]; + void SetData(const std::shared_ptr& data) { this->Array::SetData(data); - raw_values_ = values == NULLPTR ? NULLPTR : values->data(); + raw_values_ = data->GetValuesSafe(1, /*offset=*/0); } - explicit inline PrimitiveArray(const std::shared_ptr& data) { - SetData(data); - } + explicit PrimitiveArray(const std::shared_ptr& data) { SetData(data); } const uint8_t* raw_values_; }; @@ -247,7 +244,7 @@ class ARROW_EXPORT NullArray : public FlatArray { explicit NullArray(int64_t length); private: - inline void SetData(const std::shared_ptr& data) { + void SetData(const std::shared_ptr& data) { null_bitmap_data_ = NULLPTR; data->null_count = data->length; data_ = data; diff --git a/cpp/src/arrow/array/array_binary.h b/cpp/src/arrow/array/array_binary.h index 7b3e75e910ba..36016d5bb803 100644 --- a/cpp/src/arrow/array/array_binary.h +++ b/cpp/src/arrow/array/array_binary.h @@ -121,14 +121,9 @@ class BaseBinaryArray : public FlatArray { // Protected method for constructors void SetData(const std::shared_ptr& data) { - auto value_offsets = data->buffers[1]; - auto value_data = data->buffers[2]; this->Array::SetData(data); - raw_data_ = value_data == NULLPTR ? NULLPTR : value_data->data(); - raw_value_offsets_ = - value_offsets == NULLPTR - ? NULLPTR - : reinterpret_cast(value_offsets->data()); + raw_value_offsets_ = data->GetValuesSafe(1, /*offset=*/0); + raw_data_ = data->GetValuesSafe(2, /*offset=*/0); } const offset_type* raw_value_offsets_; @@ -230,7 +225,7 @@ class ARROW_EXPORT FixedSizeBinaryArray : public PrimitiveArray { const uint8_t* raw_values() const { return raw_values_ + data_->offset * byte_width_; } protected: - inline void SetData(const std::shared_ptr& data) { + void SetData(const std::shared_ptr& data) { this->PrimitiveArray::SetData(data); byte_width_ = internal::checked_cast(*type()).byte_width(); diff --git a/cpp/src/arrow/array/array_binary_test.cc b/cpp/src/arrow/array/array_binary_test.cc index 9c2cd888692b..af732eab068b 100644 --- a/cpp/src/arrow/array/array_binary_test.cc +++ b/cpp/src/arrow/array/array_binary_test.cc @@ -570,6 +570,26 @@ class TestStringBuilder : public TestBuilder { ASSERT_EQ(reps * 40, result_->value_data()->size()); } + void TestOverflowCheck() { + auto max_size = builder_->memory_limit(); + + ASSERT_OK(builder_->ValidateOverflow(1)); + ASSERT_OK(builder_->ValidateOverflow(max_size)); + ASSERT_RAISES(CapacityError, builder_->ValidateOverflow(max_size + 1)); + + ASSERT_OK(builder_->Append("bb")); + ASSERT_OK(builder_->ValidateOverflow(max_size - 2)); + ASSERT_RAISES(CapacityError, builder_->ValidateOverflow(max_size - 1)); + + ASSERT_OK(builder_->AppendNull()); + ASSERT_OK(builder_->ValidateOverflow(max_size - 2)); + ASSERT_RAISES(CapacityError, builder_->ValidateOverflow(max_size - 1)); + + ASSERT_OK(builder_->Append("ccc")); + ASSERT_OK(builder_->ValidateOverflow(max_size - 5)); + ASSERT_RAISES(CapacityError, builder_->ValidateOverflow(max_size - 4)); + } + void TestZeroLength() { // All buffers are null Done(); @@ -602,6 +622,8 @@ TYPED_TEST(TestStringBuilder, TestCapacityReserve) { this->TestCapacityReserve() TYPED_TEST(TestStringBuilder, TestZeroLength) { this->TestZeroLength(); } +TYPED_TEST(TestStringBuilder, TestOverflowCheck) { this->TestOverflowCheck(); } + // ---------------------------------------------------------------------- // ChunkedBinaryBuilder tests diff --git a/cpp/src/arrow/array/array_dict.h b/cpp/src/arrow/array/array_dict.h index c87606f7cafb..ce1f49ce5fad 100644 --- a/cpp/src/arrow/array/array_dict.h +++ b/cpp/src/arrow/array/array_dict.h @@ -75,6 +75,12 @@ class ARROW_EXPORT DictionaryArray : public Array { const std::shared_ptr& type, const std::shared_ptr& indices, const std::shared_ptr& dictionary); + static Result> FromArrays( + const std::shared_ptr& indices, const std::shared_ptr& dictionary) { + return FromArrays(::arrow::dictionary(indices->type(), dictionary->type()), indices, + dictionary); + } + /// \brief Transpose this DictionaryArray /// /// This method constructs a new dictionary array with the given dictionary diff --git a/cpp/src/arrow/array/array_dict_test.cc b/cpp/src/arrow/array/array_dict_test.cc index 2ad31121d2aa..d029838c4fc2 100644 --- a/cpp/src/arrow/array/array_dict_test.cc +++ b/cpp/src/arrow/array/array_dict_test.cc @@ -448,13 +448,14 @@ TEST(TestStringDictionaryBuilder, ArrayInit) { AssertArraysEqual(expected, *result); } -TEST(TestStringDictionaryBuilder, MakeBuilder) { - auto dict_array = ArrayFromJSON(utf8(), R"(["test", "test2"])"); - auto dict_type = dictionary(int8(), utf8()); +template +void TestStringDictionaryMakeBuilder(const std::shared_ptr& value_type) { + auto dict_array = ArrayFromJSON(value_type, R"(["test", "test2"])"); + auto dict_type = dictionary(int8(), value_type); auto int_array = ArrayFromJSON(int8(), "[0, 1, 0]"); std::unique_ptr boxed_builder; ASSERT_OK(MakeBuilder(default_memory_pool(), dict_type, &boxed_builder)); - auto& builder = checked_cast(*boxed_builder); + auto& builder = checked_cast(*boxed_builder); // Build the dictionary Array ASSERT_OK(builder.Append("test")); @@ -470,6 +471,14 @@ TEST(TestStringDictionaryBuilder, MakeBuilder) { AssertArraysEqual(expected, *result); } +TEST(TestStringDictionaryBuilder, MakeBuilder) { + TestStringDictionaryMakeBuilder>(utf8()); +} + +TEST(TestLargeStringDictionaryBuilder, MakeBuilder) { + TestStringDictionaryMakeBuilder>(large_utf8()); +} + // ARROW-4367 TEST(TestStringDictionaryBuilder, OnlyNull) { // Build the dictionary Array @@ -814,19 +823,17 @@ TEST(TestFixedSizeBinaryDictionaryBuilder, DoubleTableSize) { ASSERT_TRUE(expected.Equals(result)); } -TEST(TestFixedSizeBinaryDictionaryBuilder, InvalidTypeAppend) { +#ifndef NDEBUG +TEST(TestFixedSizeBinaryDictionaryBuilder, AppendArrayInvalidType) { // Build the dictionary Array - auto value_type = arrow::fixed_size_binary(4); + auto value_type = fixed_size_binary(4); DictionaryBuilder builder(value_type); // Build an array with different byte width - FixedSizeBinaryBuilder fsb_builder(arrow::fixed_size_binary(5)); - std::vector value{100, 1, 1, 1, 1}; - ASSERT_OK(fsb_builder.Append(value.data())); - std::shared_ptr fsb_array; - ASSERT_OK(fsb_builder.Finish(&fsb_array)); + auto fsb_array = ArrayFromJSON(fixed_size_binary(3), R"(["foo", "bar"])"); - ASSERT_RAISES(Invalid, builder.AppendArray(*fsb_array)); + ASSERT_RAISES(TypeError, builder.AppendArray(*fsb_array)); } +#endif TEST(TestDecimalDictionaryBuilder, Basic) { // Build the dictionary Array @@ -904,6 +911,78 @@ TEST(TestDecimalDictionaryBuilder, DoubleTableSize) { ASSERT_TRUE(expected.Equals(result)); } +TEST(TestNullDictionaryBuilder, Basic) { + // MakeBuilder + auto dict_type = dictionary(int8(), null()); + std::unique_ptr boxed_builder; + ASSERT_OK(MakeBuilder(default_memory_pool(), dict_type, &boxed_builder)); + auto& builder = checked_cast&>(*boxed_builder); + + ASSERT_OK(builder.AppendNull()); + ASSERT_OK(builder.AppendNull()); + ASSERT_OK(builder.AppendNull()); + ASSERT_EQ(3, builder.length()); + ASSERT_EQ(3, builder.null_count()); + + ASSERT_OK(builder.AppendNulls(4)); + ASSERT_EQ(7, builder.length()); + ASSERT_EQ(7, builder.null_count()); + + auto null_array = ArrayFromJSON(null(), "[null, null, null, null]"); + ASSERT_OK(builder.AppendArray(*null_array)); + ASSERT_EQ(11, builder.length()); + ASSERT_EQ(11, builder.null_count()); + + std::shared_ptr result; + ASSERT_OK(builder.Finish(&result)); + AssertTypeEqual(*dict_type, *result->type()); + ASSERT_EQ(11, result->length()); + ASSERT_EQ(11, result->null_count()); +} + +#ifndef NDEBUG +TEST(TestNullDictionaryBuilder, AppendArrayInvalidType) { + // MakeBuilder + auto dict_type = dictionary(int8(), null()); + std::unique_ptr boxed_builder; + ASSERT_OK(MakeBuilder(default_memory_pool(), dict_type, &boxed_builder)); + auto& builder = checked_cast&>(*boxed_builder); + + auto int8_array = ArrayFromJSON(int8(), "[0, 1, 0, null]"); + ASSERT_RAISES(TypeError, builder.AppendArray(*int8_array)); +} +#endif + +// ---------------------------------------------------------------------- +// Index byte width tests + +template +void AssertIndexByteWidth(const std::shared_ptr& value_type = + TypeTraits::type_singleton()) { + auto index_type = TypeTraits::type_singleton(); + auto dict_type = + checked_pointer_cast(dictionary(index_type, value_type)); + std::unique_ptr builder; + ASSERT_OK(MakeBuilder(default_memory_pool(), dict_type, &builder)); + auto builder_dict_type = checked_pointer_cast(builder->type()); + AssertTypeEqual(dict_type->index_type(), builder_dict_type->index_type()); +} + +typedef ::testing::Types IndexTypes; + +template +class TestDictionaryBuilderIndexByteWidth : public TestBuilder {}; + +TYPED_TEST_SUITE(TestDictionaryBuilderIndexByteWidth, IndexTypes); + +TYPED_TEST(TestDictionaryBuilderIndexByteWidth, MakeBuilder) { + AssertIndexByteWidth(); + AssertIndexByteWidth(); + AssertIndexByteWidth(); + AssertIndexByteWidth(fixed_size_binary(4)); + AssertIndexByteWidth(); +} + // ---------------------------------------------------------------------- // DictionaryArray tests diff --git a/cpp/src/arrow/array/array_list_test.cc b/cpp/src/arrow/array/array_list_test.cc index df0eb522cf4d..0dff0f48b003 100644 --- a/cpp/src/arrow/array/array_list_test.cc +++ b/cpp/src/arrow/array/array_list_test.cc @@ -467,6 +467,32 @@ class TestListArray : public TestBuilder { AssertArraysEqual(*result_, *expected); } + void TestOverflowCheck() { + Int16Builder* vb = checked_cast(builder_->value_builder()); + auto max_elements = builder_->maximum_elements(); + + ASSERT_OK(builder_->ValidateOverflow(1)); + ASSERT_OK(builder_->ValidateOverflow(max_elements)); + ASSERT_RAISES(CapacityError, builder_->ValidateOverflow(max_elements + 1)); + + ASSERT_OK(builder_->Append()); + ASSERT_OK(vb->Append(1)); + ASSERT_OK(vb->Append(2)); + ASSERT_OK(builder_->ValidateOverflow(max_elements - 2)); + ASSERT_RAISES(CapacityError, builder_->ValidateOverflow(max_elements - 1)); + + ASSERT_OK(builder_->AppendNull()); + ASSERT_OK(builder_->ValidateOverflow(max_elements - 2)); + ASSERT_RAISES(CapacityError, builder_->ValidateOverflow(max_elements - 1)); + + ASSERT_OK(builder_->Append()); + ASSERT_OK(vb->Append(1)); + ASSERT_OK(vb->Append(2)); + ASSERT_OK(vb->Append(3)); + ASSERT_OK(builder_->ValidateOverflow(max_elements - 5)); + ASSERT_RAISES(CapacityError, builder_->ValidateOverflow(max_elements - 4)); + } + protected: std::shared_ptr value_type_; @@ -508,6 +534,12 @@ TYPED_TEST(TestListArray, ValidateOffsets) { this->TestValidateOffsets(); } TYPED_TEST(TestListArray, CornerCases) { this->TestCornerCases(); } +#ifndef ARROW_LARGE_MEMORY_TESTS +TYPED_TEST(TestListArray, DISABLED_TestOverflowCheck) { this->TestOverflowCheck(); } +#else +TYPED_TEST(TestListArray, TestOverflowCheck) { this->TestOverflowCheck(); } +#endif + // ---------------------------------------------------------------------- // Map tests diff --git a/cpp/src/arrow/array/array_nested.cc b/cpp/src/arrow/array/array_nested.cc index e8d4ed92443d..97bbb18696c7 100644 --- a/cpp/src/arrow/array/array_nested.cc +++ b/cpp/src/arrow/array/array_nested.cc @@ -101,9 +101,8 @@ Status CleanListOffsets(const Array& offsets, MemoryPool* pool, } template -Result> ListArrayFromArrays(const Array& offsets, - const Array& values, - MemoryPool* pool) { +Result::ArrayType>> ListArrayFromArrays( + const Array& offsets, const Array& values, MemoryPool* pool) { using offset_type = typename TYPE::offset_type; using ArrayType = typename TypeTraits::ArrayType; using OffsetArrowType = typename CTypeTraits::ArrowType; @@ -175,6 +174,28 @@ Result> FlattenListArray(const ListArrayT& list_array, } // namespace +namespace internal { + +template +inline void SetListData(BaseListArray* self, const std::shared_ptr& data, + Type::type expected_type_id) { + ARROW_CHECK_EQ(data->buffers.size(), 2); + ARROW_CHECK_EQ(data->type->id(), expected_type_id); + ARROW_CHECK_EQ(data->child_data.size(), 1); + + self->Array::SetData(data); + + self->list_type_ = checked_cast(data->type.get()); + self->raw_value_offsets_ = + data->GetValuesSafe(1, /*offset=*/0); + + ARROW_CHECK_EQ(self->list_type_->value_type()->id(), data->child_data[0]->type->id()); + DCHECK(self->list_type_->value_type()->Equals(data->child_data[0]->type)); + self->values_ = MakeArray(self->data_->child_data[0]); +} + +} // namespace internal + ListArray::ListArray(std::shared_ptr data) { SetData(std::move(data)); } LargeListArray::LargeListArray(const std::shared_ptr& data) { SetData(data); } @@ -191,6 +212,10 @@ ListArray::ListArray(std::shared_ptr type, int64_t length, SetData(std::move(internal_data)); } +void ListArray::SetData(const std::shared_ptr& data) { + internal::SetListData(this, data); +} + LargeListArray::LargeListArray(const std::shared_ptr& type, int64_t length, const std::shared_ptr& value_offsets, const std::shared_ptr& values, @@ -203,50 +228,19 @@ LargeListArray::LargeListArray(const std::shared_ptr& type, int64_t le SetData(internal_data); } -void ListArray::SetData(const std::shared_ptr& data, - Type::type expected_type_id) { - this->Array::SetData(data); - ARROW_CHECK_EQ(data->buffers.size(), 2); - ARROW_CHECK_EQ(data->type->id(), expected_type_id); - list_type_ = checked_cast(data->type.get()); - - auto value_offsets = data->buffers[1]; - raw_value_offsets_ = value_offsets == nullptr - ? nullptr - : reinterpret_cast(value_offsets->data()); - - ARROW_CHECK_EQ(data_->child_data.size(), 1); - ARROW_CHECK_EQ(list_type_->value_type()->id(), data->child_data[0]->type->id()); - DCHECK(list_type_->value_type()->Equals(data->child_data[0]->type)); - values_ = MakeArray(data_->child_data[0]); -} - void LargeListArray::SetData(const std::shared_ptr& data) { - this->Array::SetData(data); - ARROW_CHECK_EQ(data->buffers.size(), 2); - ARROW_CHECK_EQ(data->type->id(), Type::LARGE_LIST); - list_type_ = checked_cast(data->type.get()); - - auto value_offsets = data->buffers[1]; - raw_value_offsets_ = value_offsets == nullptr - ? nullptr - : reinterpret_cast(value_offsets->data()); - - ARROW_CHECK_EQ(data_->child_data.size(), 1); - ARROW_CHECK_EQ(list_type_->value_type()->id(), data->child_data[0]->type->id()); - DCHECK(list_type_->value_type()->Equals(data->child_data[0]->type)); - values_ = MakeArray(data_->child_data[0]); + internal::SetListData(this, data); } -Result> ListArray::FromArrays(const Array& offsets, - const Array& values, - MemoryPool* pool) { +Result> ListArray::FromArrays(const Array& offsets, + const Array& values, + MemoryPool* pool) { return ListArrayFromArrays(offsets, values, pool); } -Result> LargeListArray::FromArrays(const Array& offsets, - const Array& values, - MemoryPool* pool) { +Result> LargeListArray::FromArrays(const Array& offsets, + const Array& values, + MemoryPool* pool) { return ListArrayFromArrays(offsets, values, pool); } @@ -381,7 +375,7 @@ Status MapArray::ValidateChildData( void MapArray::SetData(const std::shared_ptr& data) { ARROW_CHECK_OK(ValidateChildData(data->child_data)); - this->ListArray::SetData(data, Type::MAP); + internal::SetListData(this, data, Type::MAP); map_type_ = checked_cast(data->type.get()); const auto& pair_data = data->child_data[0]; keys_ = MakeArray(pair_data->child_data[0]); @@ -601,10 +595,7 @@ void UnionArray::SetData(std::shared_ptr data) { union_type_ = checked_cast(data_->type.get()); ARROW_CHECK_GE(data_->buffers.size(), 2); - auto type_codes = data_->buffers[1]; - raw_type_codes_ = type_codes == nullptr - ? nullptr - : reinterpret_cast(type_codes->data()); + raw_type_codes_ = data->GetValuesSafe(1, /*offset=*/0); boxed_fields_.resize(data_->child_data.size()); } @@ -626,10 +617,7 @@ void DenseUnionArray::SetData(const std::shared_ptr& data) { // No validity bitmap ARROW_CHECK_EQ(data_->buffers[0], nullptr); - auto value_offsets = data_->buffers[2]; - raw_value_offsets_ = value_offsets == nullptr - ? nullptr - : reinterpret_cast(value_offsets->data()); + raw_value_offsets_ = data->GetValuesSafe(2, /*offset=*/0); } SparseUnionArray::SparseUnionArray(std::shared_ptr data) { diff --git a/cpp/src/arrow/array/array_nested.h b/cpp/src/arrow/array/array_nested.h index e37c34babc10..d39f33f47025 100644 --- a/cpp/src/arrow/array/array_nested.h +++ b/cpp/src/arrow/array/array_nested.h @@ -41,6 +41,20 @@ namespace arrow { // ---------------------------------------------------------------------- // ListArray +template +class BaseListArray; + +namespace internal { + +// Private helper for ListArray::SetData. +// Unfortunately, trying to define BaseListArray::SetData outside of this header +// doesn't play well with MSVC. +template +void SetListData(BaseListArray* self, const std::shared_ptr& data, + Type::type expected_type_id = TYPE::type_id); + +} // namespace internal + /// Base class for variable-sized list arrays, regardless of offset size. template class BaseListArray : public Array { @@ -76,6 +90,10 @@ class BaseListArray : public Array { } protected: + friend void internal::SetListData(BaseListArray* self, + const std::shared_ptr& data, + Type::type expected_type_id); + const TypeClass* list_type_ = NULLPTR; std::shared_ptr values_; const offset_type* raw_value_offsets_ = NULLPTR; @@ -103,7 +121,7 @@ class ARROW_EXPORT ListArray : public BaseListArray { /// \param[in] values Array containing list values /// \param[in] pool MemoryPool in case new offsets array needs to be /// allocated because of null values - static Result> FromArrays( + static Result> FromArrays( const Array& offsets, const Array& values, MemoryPool* pool = default_memory_pool()); @@ -121,8 +139,8 @@ class ARROW_EXPORT ListArray : public BaseListArray { protected: // This constructor defers SetData to a derived array class ListArray() = default; - void SetData(const std::shared_ptr& data, - Type::type expected_type_id = Type::LIST); + + void SetData(const std::shared_ptr& data); }; /// Concrete Array class for large list data (with 64-bit offsets) @@ -148,7 +166,7 @@ class ARROW_EXPORT LargeListArray : public BaseListArray { /// \param[in] values Array containing list values /// \param[in] pool MemoryPool in case new offsets array needs to be /// allocated because of null values - static Result> FromArrays( + static Result> FromArrays( const Array& offsets, const Array& values, MemoryPool* pool = default_memory_pool()); @@ -310,8 +328,7 @@ class ARROW_EXPORT StructArray : public Array { /// The length and data type are automatically inferred from the arguments. /// There should be at least one child array. static Result> Make( - const std::vector>& children, - const std::vector& field_names, + const ArrayVector& children, const std::vector& field_names, std::shared_ptr null_bitmap = NULLPTR, int64_t null_count = kUnknownNullCount, int64_t offset = 0); @@ -321,8 +338,7 @@ class ARROW_EXPORT StructArray : public Array { /// There should be at least one child array. This method does not /// check that field types and child array types are consistent. static Result> Make( - const std::vector>& children, - const std::vector>& fields, + const ArrayVector& children, const FieldVector& fields, std::shared_ptr null_bitmap = NULLPTR, int64_t null_count = kUnknownNullCount, int64_t offset = 0); diff --git a/cpp/src/arrow/array/array_primitive.h b/cpp/src/arrow/array/array_primitive.h index c58fee77ceff..8e5c1eac2829 100644 --- a/cpp/src/arrow/array/array_primitive.h +++ b/cpp/src/arrow/array/array_primitive.h @@ -119,11 +119,6 @@ class ARROW_EXPORT DayTimeIntervalArray : public PrimitiveArray { int32_t byte_width() const { return sizeof(TypeClass::DayMilliseconds); } const uint8_t* raw_values() const { return raw_values_ + data_->offset * byte_width(); } - - protected: - inline void SetData(const std::shared_ptr& data) { - this->PrimitiveArray::SetData(data); - } }; } // namespace arrow diff --git a/cpp/src/arrow/array/array_test.cc b/cpp/src/arrow/array/array_test.cc index 2702c355c013..ed0fe4ac8bf3 100644 --- a/cpp/src/arrow/array/array_test.cc +++ b/cpp/src/arrow/array/array_test.cc @@ -234,6 +234,9 @@ TEST_F(TestArray, SliceRecomputeNullCount) { slice = array->Slice(4); ASSERT_EQ(4, slice->null_count()); + auto slice2 = slice->Slice(0); + ASSERT_EQ(4, slice2->null_count()); + slice = array->Slice(0); ASSERT_EQ(5, slice->null_count()); @@ -1501,7 +1504,7 @@ void CheckFloatingNanEquality() { // NaN != non-NaN ArrayFromVector(type, {false, true}, {0.5, nan_value}, &a); - ArrayFromVector(type, {false, true}, {0.5, 0.0}, &a); + ArrayFromVector(type, {false, true}, {0.5, 0.0}, &b); ASSERT_FALSE(a->Equals(b)); ASSERT_FALSE(b->Equals(a)); ASSERT_FALSE(a->Equals(b, EqualOptions().nans_equal(true))); @@ -1520,6 +1523,73 @@ void CheckFloatingNanEquality() { ASSERT_TRUE(b->RangeEquals(a, 0, 1, 0)); } +template +void CheckFloatingInfinityEquality() { + std::shared_ptr a, b; + std::shared_ptr type = TypeTraits::type_singleton(); + + const auto infinity = std::numeric_limits::infinity(); + + for (auto nans_equal : {false, true}) { + // Infinity in a null entry + ArrayFromVector(type, {true, false}, {0.5, infinity}, &a); + ArrayFromVector(type, {true, false}, {0.5, -infinity}, &b); + ASSERT_TRUE(a->Equals(b)); + ASSERT_TRUE(b->Equals(a)); + ASSERT_TRUE(a->ApproxEquals(b, EqualOptions().atol(1e-5).nans_equal(nans_equal))); + ASSERT_TRUE(b->ApproxEquals(a, EqualOptions().atol(1e-5).nans_equal(nans_equal))); + ASSERT_TRUE(a->RangeEquals(b, 0, 2, 0)); + ASSERT_TRUE(b->RangeEquals(a, 0, 2, 0)); + ASSERT_TRUE(a->RangeEquals(b, 1, 2, 1)); + ASSERT_TRUE(b->RangeEquals(a, 1, 2, 1)); + + // Infinity in a valid entry + ArrayFromVector(type, {false, true}, {0.5, infinity}, &a); + ArrayFromVector(type, {false, true}, {0.5, infinity}, &b); + ASSERT_TRUE(a->Equals(b)); + ASSERT_TRUE(b->Equals(a)); + ASSERT_TRUE(a->ApproxEquals(b, EqualOptions().atol(1e-5).nans_equal(nans_equal))); + ASSERT_TRUE(b->ApproxEquals(a, EqualOptions().atol(1e-5).nans_equal(nans_equal))); + ASSERT_TRUE(a->ApproxEquals(b, EqualOptions().atol(1e-5).nans_equal(nans_equal))); + ASSERT_TRUE(b->ApproxEquals(a, EqualOptions().atol(1e-5).nans_equal(nans_equal))); + // Infinity in tested range + ASSERT_TRUE(a->RangeEquals(b, 0, 2, 0)); + ASSERT_TRUE(b->RangeEquals(a, 0, 2, 0)); + ASSERT_TRUE(a->RangeEquals(b, 1, 2, 1)); + ASSERT_TRUE(b->RangeEquals(a, 1, 2, 1)); + // Infinity not in tested range + ASSERT_TRUE(a->RangeEquals(b, 0, 1, 0)); + ASSERT_TRUE(b->RangeEquals(a, 0, 1, 0)); + + // Infinity != non-infinity + ArrayFromVector(type, {false, true}, {0.5, -infinity}, &a); + ArrayFromVector(type, {false, true}, {0.5, 0.0}, &b); + ASSERT_FALSE(a->Equals(b)); + ASSERT_FALSE(b->Equals(a)); + ASSERT_FALSE(a->ApproxEquals(b, EqualOptions().atol(1e-5).nans_equal(nans_equal))); + ASSERT_FALSE(b->ApproxEquals(a)); + ASSERT_FALSE(a->ApproxEquals(b, EqualOptions().atol(1e-5).nans_equal(nans_equal))); + ASSERT_FALSE(b->ApproxEquals(a, EqualOptions().atol(1e-5).nans_equal(nans_equal))); + // Infinity != Negative infinity + ArrayFromVector(type, {true, true}, {0.5, -infinity}, &a); + ArrayFromVector(type, {true, true}, {0.5, infinity}, &b); + ASSERT_FALSE(a->Equals(b)); + ASSERT_FALSE(b->Equals(a)); + ASSERT_FALSE(a->ApproxEquals(b)); + ASSERT_FALSE(b->ApproxEquals(a)); + ASSERT_FALSE(a->ApproxEquals(b, EqualOptions().atol(1e-5).nans_equal(nans_equal))); + ASSERT_FALSE(b->ApproxEquals(a, EqualOptions().atol(1e-5).nans_equal(nans_equal))); + // Infinity in tested range + ASSERT_FALSE(a->RangeEquals(b, 0, 2, 0)); + ASSERT_FALSE(b->RangeEquals(a, 0, 2, 0)); + ASSERT_FALSE(a->RangeEquals(b, 1, 2, 1)); + ASSERT_FALSE(b->RangeEquals(a, 1, 2, 1)); + // Infinity not in tested range + ASSERT_TRUE(a->RangeEquals(b, 0, 1, 0)); + ASSERT_TRUE(b->RangeEquals(a, 0, 1, 0)); + } +} + TEST(TestPrimitiveAdHoc, FloatingApproxEquals) { CheckApproxEquals(); CheckApproxEquals(); @@ -1535,6 +1605,11 @@ TEST(TestPrimitiveAdHoc, FloatingNanEquality) { CheckFloatingNanEquality(); } +TEST(TestPrimitiveAdHoc, FloatingInfinityEquality) { + CheckFloatingInfinityEquality(); + CheckFloatingInfinityEquality(); +} + // ---------------------------------------------------------------------- // FixedSizeBinary tests @@ -2029,6 +2104,19 @@ TEST_F(TestAdaptiveIntBuilder, TestAppendNulls) { } } +TEST(TestAdaptiveIntBuilderWithStartIntSize, TestReset) { + auto builder = std::make_shared( + static_cast(sizeof(int16_t)), default_memory_pool()); + AssertTypeEqual(*int16(), *builder->type()); + + ASSERT_OK( + builder->Append(static_cast(std::numeric_limits::max()) + 1)); + AssertTypeEqual(*int32(), *builder->type()); + + builder->Reset(); + AssertTypeEqual(*int16(), *builder->type()); +} + class TestAdaptiveUIntBuilder : public TestBuilder { public: void SetUp() { @@ -2234,6 +2322,19 @@ TEST_F(TestAdaptiveUIntBuilder, TestAppendNulls) { } } +TEST(TestAdaptiveUIntBuilderWithStartIntSize, TestReset) { + auto builder = std::make_shared( + static_cast(sizeof(uint16_t)), default_memory_pool()); + AssertTypeEqual(uint16(), builder->type()); + + ASSERT_OK( + builder->Append(static_cast(std::numeric_limits::max()) + 1)); + AssertTypeEqual(uint32(), builder->type()); + + builder->Reset(); + AssertTypeEqual(uint16(), builder->type()); +} + // ---------------------------------------------------------------------- // Test Decimal arrays diff --git a/cpp/src/arrow/array/array_view_test.cc b/cpp/src/arrow/array/array_view_test.cc index 3aac62da0dee..e73bbda7abc5 100644 --- a/cpp/src/arrow/array/array_view_test.cc +++ b/cpp/src/arrow/array/array_view_test.cc @@ -329,6 +329,17 @@ TEST(TestArrayView, FixedSizeListAsFlat) { // XXX With nulls (currently fails) } +TEST(TestArrayView, FixedSizeListAsFixedSizeBinary) { + auto ty1 = fixed_size_list(int32(), 1); +#if ARROW_LITTLE_ENDIAN + auto arr = ArrayFromJSON(ty1, "[[2020568934], [2054316386]]"); +#else + auto arr = ArrayFromJSON(ty1, "[[1718579064], [1650553466]]"); +#endif + auto expected = ArrayFromJSON(fixed_size_binary(4), R"(["foox", "barz"])"); + CheckView(arr, expected); +} + TEST(TestArrayView, SparseUnionAsStruct) { auto child1 = ArrayFromJSON(int16(), "[0, -1, 42]"); auto child2 = ArrayFromJSON(int32(), "[0, 1069547520, -1071644672]"); diff --git a/cpp/src/arrow/array/builder_adaptive.cc b/cpp/src/arrow/array/builder_adaptive.cc index 47880e91663d..36e5546a749a 100644 --- a/cpp/src/arrow/array/builder_adaptive.cc +++ b/cpp/src/arrow/array/builder_adaptive.cc @@ -33,7 +33,8 @@ namespace arrow { using internal::AdaptiveIntBuilderBase; -AdaptiveIntBuilderBase::AdaptiveIntBuilderBase(MemoryPool* pool) : ArrayBuilder(pool) {} +AdaptiveIntBuilderBase::AdaptiveIntBuilderBase(uint8_t start_int_size, MemoryPool* pool) + : ArrayBuilder(pool), start_int_size_(start_int_size), int_size_(start_int_size) {} void AdaptiveIntBuilderBase::Reset() { ArrayBuilder::Reset(); @@ -41,7 +42,7 @@ void AdaptiveIntBuilderBase::Reset() { raw_data_ = nullptr; pending_pos_ = 0; pending_has_nulls_ = false; - int_size_ = sizeof(uint8_t); + int_size_ = start_int_size_; } Status AdaptiveIntBuilderBase::Resize(int64_t capacity) { @@ -124,7 +125,8 @@ std::shared_ptr AdaptiveIntBuilder::type() const { return nullptr; } -AdaptiveIntBuilder::AdaptiveIntBuilder(MemoryPool* pool) : AdaptiveIntBuilderBase(pool) {} +AdaptiveIntBuilder::AdaptiveIntBuilder(uint8_t start_int_size, MemoryPool* pool) + : AdaptiveIntBuilderBase(start_int_size, pool) {} Status AdaptiveIntBuilder::FinishInternal(std::shared_ptr* out) { RETURN_NOT_OK(CommitPendingData()); @@ -264,8 +266,8 @@ Status AdaptiveIntBuilder::ExpandIntSize(uint8_t new_int_size) { return Status::OK(); } -AdaptiveUIntBuilder::AdaptiveUIntBuilder(MemoryPool* pool) - : AdaptiveIntBuilderBase(pool) {} +AdaptiveUIntBuilder::AdaptiveUIntBuilder(uint8_t start_int_size, MemoryPool* pool) + : AdaptiveIntBuilderBase(start_int_size, pool) {} Status AdaptiveUIntBuilder::FinishInternal(std::shared_ptr* out) { RETURN_NOT_OK(CommitPendingData()); diff --git a/cpp/src/arrow/array/builder_adaptive.h b/cpp/src/arrow/array/builder_adaptive.h index 1bce339433dd..1895f8ddaee5 100644 --- a/cpp/src/arrow/array/builder_adaptive.h +++ b/cpp/src/arrow/array/builder_adaptive.h @@ -35,7 +35,10 @@ namespace internal { class ARROW_EXPORT AdaptiveIntBuilderBase : public ArrayBuilder { public: - explicit AdaptiveIntBuilderBase(MemoryPool* pool); + AdaptiveIntBuilderBase(uint8_t start_int_size, MemoryPool* pool); + + explicit AdaptiveIntBuilderBase(MemoryPool* pool) + : AdaptiveIntBuilderBase(sizeof(uint8_t), pool) {} /// \brief Append multiple nulls /// \param[in] length the number of nulls to append @@ -88,7 +91,9 @@ class ARROW_EXPORT AdaptiveIntBuilderBase : public ArrayBuilder { std::shared_ptr data_; uint8_t* raw_data_ = NULLPTR; - uint8_t int_size_ = sizeof(uint8_t); + + const uint8_t start_int_size_; + uint8_t int_size_; static constexpr int32_t pending_size_ = 1024; uint8_t pending_valid_[pending_size_]; @@ -101,7 +106,11 @@ class ARROW_EXPORT AdaptiveIntBuilderBase : public ArrayBuilder { class ARROW_EXPORT AdaptiveUIntBuilder : public internal::AdaptiveIntBuilderBase { public: - explicit AdaptiveUIntBuilder(MemoryPool* pool = default_memory_pool()); + explicit AdaptiveUIntBuilder(uint8_t start_int_size, + MemoryPool* pool = default_memory_pool()); + + explicit AdaptiveUIntBuilder(MemoryPool* pool = default_memory_pool()) + : AdaptiveUIntBuilder(sizeof(uint8_t), pool) {} using ArrayBuilder::Advance; using internal::AdaptiveIntBuilderBase::Reset; @@ -135,7 +144,11 @@ class ARROW_EXPORT AdaptiveUIntBuilder : public internal::AdaptiveIntBuilderBase class ARROW_EXPORT AdaptiveIntBuilder : public internal::AdaptiveIntBuilderBase { public: - explicit AdaptiveIntBuilder(MemoryPool* pool = default_memory_pool()); + explicit AdaptiveIntBuilder(uint8_t start_int_size, + MemoryPool* pool = default_memory_pool()); + + explicit AdaptiveIntBuilder(MemoryPool* pool = default_memory_pool()) + : AdaptiveIntBuilder(sizeof(uint8_t), pool) {} using ArrayBuilder::Advance; using internal::AdaptiveIntBuilderBase::Reset; diff --git a/cpp/src/arrow/array/builder_base.cc b/cpp/src/arrow/array/builder_base.cc index 4c21859fae3b..b92cc2858946 100644 --- a/cpp/src/arrow/array/builder_base.cc +++ b/cpp/src/arrow/array/builder_base.cc @@ -29,6 +29,22 @@ namespace arrow { +Status ArrayBuilder::CheckArrayType(const std::shared_ptr& expected_type, + const Array& array, const char* message) { + if (!expected_type->Equals(*array.type())) { + return Status::TypeError(message); + } + return Status::OK(); +} + +Status ArrayBuilder::CheckArrayType(Type::type expected_type, const Array& array, + const char* message) { + if (array.type_id() != expected_type) { + return Status::TypeError(message); + } + return Status::OK(); +} + Status ArrayBuilder::TrimBuffer(const int64_t bytes_filled, ResizableBuffer* buffer) { if (buffer) { if (bytes_filled < buffer->size()) { @@ -83,6 +99,12 @@ Status ArrayBuilder::Finish(std::shared_ptr* out) { return Status::OK(); } +Result> ArrayBuilder::Finish() { + std::shared_ptr out; + RETURN_NOT_OK(Finish(&out)); + return out; +} + void ArrayBuilder::Reset() { capacity_ = length_ = null_count_ = 0; null_bitmap_builder_.Reset(); diff --git a/cpp/src/arrow/array/builder_base.h b/cpp/src/arrow/array/builder_base.h index 8d327b713b66..d73681756bab 100644 --- a/cpp/src/arrow/array/builder_base.h +++ b/cpp/src/arrow/array/builder_base.h @@ -56,6 +56,8 @@ class ARROW_EXPORT ArrayBuilder { /// skip shared pointers and just return a raw pointer ArrayBuilder* child(int i) { return children_[i].get(); } + const std::shared_ptr& child_builder(int i) const { return children_[i]; } + int num_children() const { return static_cast(children_.size()); } virtual int64_t length() const { return length_; } @@ -118,6 +120,13 @@ class ARROW_EXPORT ArrayBuilder { /// \return Status Status Finish(std::shared_ptr* out); + /// \brief Return result of builder as an Array object. + /// + /// The builder is reset except for DictionaryBuilder. + /// + /// \return The finalized Array object + Result> Finish(); + /// \brief Return the type of the built Array virtual std::shared_ptr type() const = 0; @@ -200,6 +209,12 @@ class ARROW_EXPORT ArrayBuilder { return Status::OK(); } + // Check for array type + Status CheckArrayType(const std::shared_ptr& expected_type, + const Array& array, const char* message); + Status CheckArrayType(Type::type expected_type, const Array& array, + const char* message); + MemoryPool* pool_; TypedBufferBuilder null_bitmap_builder_; diff --git a/cpp/src/arrow/array/builder_binary.h b/cpp/src/arrow/array/builder_binary.h index 593b533a19c8..70fde2a2e646 100644 --- a/cpp/src/arrow/array/builder_binary.h +++ b/cpp/src/arrow/array/builder_binary.h @@ -78,9 +78,7 @@ class BaseBinaryBuilder : public ArrayBuilder { Status AppendNulls(int64_t length) final { const int64_t num_bytes = value_data_builder_.length(); - if (ARROW_PREDICT_FALSE(num_bytes > memory_limit())) { - return AppendOverflow(num_bytes); - } + ARROW_RETURN_NOT_OK(ValidateOverflow(0)); ARROW_RETURN_NOT_OK(Reserve(length)); for (int64_t i = 0; i < length; ++i) { offsets_builder_.UnsafeAppend(static_cast(num_bytes)); @@ -232,6 +230,16 @@ class BaseBinaryBuilder : public ArrayBuilder { value_data_builder_.Reset(); } + Status ValidateOverflow(int64_t new_bytes) { + auto new_size = value_data_builder_.length() + new_bytes; + if (ARROW_PREDICT_FALSE(new_size > memory_limit())) { + return Status::CapacityError("array cannot contain more than ", memory_limit(), + " bytes, have ", new_size); + } else { + return Status::OK(); + } + } + Status Resize(int64_t capacity) override { // XXX Why is this check necessary? There is no reason to disallow, say, // binary arrays with more than 2**31 empty or null values. @@ -249,12 +257,8 @@ class BaseBinaryBuilder : public ArrayBuilder { /// \brief Ensures there is enough allocated capacity to append the indicated /// number of bytes to the value data buffer without additional allocations Status ReserveData(int64_t elements) { - const int64_t size = value_data_length() + elements; - ARROW_RETURN_IF(size > memory_limit(), - Status::CapacityError("Cannot reserve capacity larger than ", - memory_limit(), " bytes")); - return (size > value_data_capacity()) ? value_data_builder_.Reserve(elements) - : Status::OK(); + ARROW_RETURN_NOT_OK(ValidateOverflow(elements)); + return value_data_builder_.Reserve(elements); } Status FinishInternal(std::shared_ptr* out) override { @@ -317,16 +321,9 @@ class BaseBinaryBuilder : public ArrayBuilder { TypedBufferBuilder offsets_builder_; TypedBufferBuilder value_data_builder_; - Status AppendOverflow(int64_t num_bytes) { - return Status::CapacityError("array cannot contain more than ", memory_limit(), - " bytes, have ", num_bytes); - } - Status AppendNextOffset() { const int64_t num_bytes = value_data_builder_.length(); - if (ARROW_PREDICT_FALSE(num_bytes > memory_limit())) { - return AppendOverflow(num_bytes); - } + ARROW_RETURN_NOT_OK(ValidateOverflow(0)); return offsets_builder_.Append(static_cast(num_bytes)); } @@ -450,6 +447,10 @@ class ARROW_EXPORT FixedSizeBinaryBuilder : public ArrayBuilder { } } + void UnsafeAppend(const char* value) { + UnsafeAppend(reinterpret_cast(value)); + } + void UnsafeAppend(util::string_view value) { #ifndef NDEBUG CheckValueSize(static_cast(value.size())); @@ -462,6 +463,23 @@ class ARROW_EXPORT FixedSizeBinaryBuilder : public ArrayBuilder { byte_builder_.UnsafeAppend(/*num_copies=*/byte_width_, 0); } + Status ValidateOverflow(int64_t new_bytes) const { + auto new_size = byte_builder_.length() + new_bytes; + if (ARROW_PREDICT_FALSE(new_size > memory_limit())) { + return Status::CapacityError("array cannot contain more than ", memory_limit(), + " bytes, have ", new_size); + } else { + return Status::OK(); + } + } + + /// \brief Ensures there is enough allocated capacity to append the indicated + /// number of bytes to the value data buffer without additional allocations + Status ReserveData(int64_t elements) { + ARROW_RETURN_NOT_OK(ValidateOverflow(elements)); + return byte_builder_.Reserve(elements); + } + void Reset() override; Status Resize(int64_t capacity) override; Status FinishInternal(std::shared_ptr* out) override; diff --git a/cpp/src/arrow/array/builder_dict.cc b/cpp/src/arrow/array/builder_dict.cc index 54fd94856eaf..e2a758f4419f 100644 --- a/cpp/src/arrow/array/builder_dict.cc +++ b/cpp/src/arrow/array/builder_dict.cc @@ -18,6 +18,7 @@ #include "arrow/array/builder_dict.h" #include +#include #include "arrow/array/dict_internal.h" #include "arrow/status.h" @@ -68,21 +69,20 @@ class DictionaryMemoTable::DictionaryMemoTableImpl { } private: - template - enable_if_no_memoize InsertValues(const DType& type, - const ArrayType&) { + template + enable_if_no_memoize InsertValues(const T& type, const ArrayType&) { return Status::NotImplemented("Inserting array values of ", type, " is not implemented"); } - template - enable_if_memoize InsertValues(const DType&, const ArrayType& array) { + template + enable_if_memoize InsertValues(const T&, const ArrayType& array) { if (array.null_count() > 0) { return Status::Invalid("Cannot insert dictionary values containing nulls"); } for (int64_t i = 0; i < array.length(); ++i) { int32_t unused_memo_index; - RETURN_NOT_OK(impl_->GetOrInsert(array.GetView(i), &unused_memo_index)); + RETURN_NOT_OK(impl_->GetOrInsert(array.GetView(i), &unused_memo_index)); } return Status::OK(); } @@ -112,8 +112,8 @@ class DictionaryMemoTable::DictionaryMemoTableImpl { }; public: - DictionaryMemoTableImpl(MemoryPool* pool, const std::shared_ptr& type) - : pool_(pool), type_(type), memo_table_(nullptr) { + DictionaryMemoTableImpl(MemoryPool* pool, std::shared_ptr type) + : pool_(pool), type_(std::move(type)), memo_table_(nullptr) { MemoTableInitializer visitor{type_, pool_, &memo_table_}; ARROW_CHECK_OK(VisitTypeInline(*type_, &visitor)); } @@ -127,9 +127,10 @@ class DictionaryMemoTable::DictionaryMemoTableImpl { return VisitTypeInline(*array.type(), &visitor); } - template - Status GetOrInsert(const T& value, int32_t* out) { - using ConcreteMemoTable = typename DictionaryCTraits::MemoTableType; + template ::type> + Status GetOrInsert(CType value, int32_t* out) { + using ConcreteMemoTable = typename DictionaryTraits::MemoTableType; return checked_cast(memo_table_.get())->GetOrInsert(value, out); } @@ -158,9 +159,10 @@ DictionaryMemoTable::DictionaryMemoTable(MemoryPool* pool, DictionaryMemoTable::~DictionaryMemoTable() = default; -#define GET_OR_INSERT(C_TYPE) \ - Status DictionaryMemoTable::GetOrInsert(C_TYPE value, int32_t* out) { \ - return impl_->GetOrInsert(value, out); \ +#define GET_OR_INSERT(C_TYPE) \ + Status DictionaryMemoTable::GetOrInsert( \ + const typename CTypeTraits::ArrowType*, C_TYPE value, int32_t* out) { \ + return impl_->GetOrInsert::ArrowType>(value, out); \ } GET_OR_INSERT(bool) @@ -174,10 +176,19 @@ GET_OR_INSERT(uint32_t) GET_OR_INSERT(uint64_t) GET_OR_INSERT(float) GET_OR_INSERT(double) -GET_OR_INSERT(util::string_view) #undef GET_OR_INSERT +Status DictionaryMemoTable::GetOrInsert(const BinaryType*, util::string_view value, + int32_t* out) { + return impl_->GetOrInsert(value, out); +} + +Status DictionaryMemoTable::GetOrInsert(const LargeBinaryType*, util::string_view value, + int32_t* out) { + return impl_->GetOrInsert(value, out); +} + Status DictionaryMemoTable::GetArrayData(int64_t start_offset, std::shared_ptr* out) { return impl_->GetArrayData(start_offset, out); diff --git a/cpp/src/arrow/array/builder_dict.h b/cpp/src/arrow/array/builder_dict.h index 9a0f268c4d02..7f920bb8cd60 100644 --- a/cpp/src/arrow/array/builder_dict.h +++ b/cpp/src/arrow/array/builder_dict.h @@ -32,6 +32,8 @@ #include "arrow/status.h" #include "arrow/type.h" #include "arrow/type_traits.h" +#include "arrow/util/checked_cast.h" +#include "arrow/util/decimal.h" #include "arrow/util/macros.h" #include "arrow/util/visibility.h" @@ -42,24 +44,24 @@ namespace arrow { namespace internal { -template -struct DictionaryScalar { +template +struct DictionaryValue { using type = typename T::c_type; + using PhysicalType = T; }; -template <> -struct DictionaryScalar { - using type = util::string_view; -}; - -template <> -struct DictionaryScalar { +template +struct DictionaryValue> { using type = util::string_view; + using PhysicalType = + typename std::conditional::value, + BinaryType, LargeBinaryType>::type; }; -template <> -struct DictionaryScalar { +template +struct DictionaryValue> { using type = util::string_view; + using PhysicalType = BinaryType; }; class ARROW_EXPORT DictionaryMemoTable { @@ -68,19 +70,6 @@ class ARROW_EXPORT DictionaryMemoTable { DictionaryMemoTable(MemoryPool* pool, const std::shared_ptr& dictionary); ~DictionaryMemoTable(); - Status GetOrInsert(bool value, int32_t* out); - Status GetOrInsert(int8_t value, int32_t* out); - Status GetOrInsert(int16_t value, int32_t* out); - Status GetOrInsert(int32_t value, int32_t* out); - Status GetOrInsert(int64_t value, int32_t* out); - Status GetOrInsert(uint8_t value, int32_t* out); - Status GetOrInsert(uint16_t value, int32_t* out); - Status GetOrInsert(uint32_t value, int32_t* out); - Status GetOrInsert(uint64_t value, int32_t* out); - Status GetOrInsert(float value, int32_t* out); - Status GetOrInsert(double value, int32_t* out); - Status GetOrInsert(util::string_view value, int32_t* out); - Status GetArrayData(int64_t start_offset, std::shared_ptr* out); /// \brief Insert new memo values @@ -88,7 +77,31 @@ class ARROW_EXPORT DictionaryMemoTable { int32_t size() const; + template + Status GetOrInsert(typename DictionaryValue::type value, int32_t* out) { + // We want to keep the DictionaryMemoTable implementation private, also we can't + // use extern template classes because of compiler issues (MinGW?). Instead, + // we expose explicit function overrides for each supported physical type. + const typename DictionaryValue::PhysicalType* physical_type = NULLPTR; + return GetOrInsert(physical_type, value, out); + } + private: + Status GetOrInsert(const BooleanType*, bool value, int32_t* out); + Status GetOrInsert(const Int8Type*, int8_t value, int32_t* out); + Status GetOrInsert(const Int16Type*, int16_t value, int32_t* out); + Status GetOrInsert(const Int32Type*, int32_t value, int32_t* out); + Status GetOrInsert(const Int64Type*, int64_t value, int32_t* out); + Status GetOrInsert(const UInt8Type*, uint8_t value, int32_t* out); + Status GetOrInsert(const UInt16Type*, uint16_t value, int32_t* out); + Status GetOrInsert(const UInt32Type*, uint32_t value, int32_t* out); + Status GetOrInsert(const UInt64Type*, uint64_t value, int32_t* out); + Status GetOrInsert(const FloatType*, float value, int32_t* out); + Status GetOrInsert(const DoubleType*, double value, int32_t* out); + + Status GetOrInsert(const BinaryType*, util::string_view value, int32_t* out); + Status GetOrInsert(const LargeBinaryType*, util::string_view value, int32_t* out); + class DictionaryMemoTableImpl; std::unique_ptr impl_; }; @@ -101,15 +114,30 @@ class ARROW_EXPORT DictionaryMemoTable { template class DictionaryBuilderBase : public ArrayBuilder { public: - using Scalar = typename DictionaryScalar::type; + using TypeClass = DictionaryType; + using Value = typename DictionaryValue::type; // WARNING: the type given below is the value type, not the DictionaryType. // The DictionaryType is instantiated on the Finish() call. - template - DictionaryBuilderBase(enable_if_t::value, + template + DictionaryBuilderBase(uint8_t start_int_size, + enable_if_t::value && + !is_fixed_size_binary_type::value, const std::shared_ptr&> value_type, MemoryPool* pool = default_memory_pool()) + : ArrayBuilder(pool), + memo_table_(new internal::DictionaryMemoTable(pool, value_type)), + delta_offset_(0), + byte_width_(-1), + indices_builder_(start_int_size, pool), + value_type_(value_type) {} + + template + explicit DictionaryBuilderBase( + enable_if_t::value, const std::shared_ptr&> + value_type, + MemoryPool* pool = default_memory_pool()) : ArrayBuilder(pool), memo_table_(new internal::DictionaryMemoTable(pool, value_type)), delta_offset_(0), @@ -117,6 +145,20 @@ class DictionaryBuilderBase : public ArrayBuilder { indices_builder_(pool), value_type_(value_type) {} + template + DictionaryBuilderBase(uint8_t start_int_size, + enable_if_t::value && + is_fixed_size_binary_type::value, + const std::shared_ptr&> + value_type, + MemoryPool* pool = default_memory_pool()) + : ArrayBuilder(pool), + memo_table_(new internal::DictionaryMemoTable(pool, value_type)), + delta_offset_(0), + byte_width_(static_cast(*value_type).byte_width()), + indices_builder_(start_int_size, pool), + value_type_(value_type) {} + template explicit DictionaryBuilderBase( enable_if_fixed_size_binary&> value_type, @@ -134,8 +176,8 @@ class DictionaryBuilderBase : public ArrayBuilder { : DictionaryBuilderBase(TypeTraits::type_singleton(), pool) {} // This constructor doesn't check for errors. Use InsertMemoValues instead. - DictionaryBuilderBase(const std::shared_ptr& dictionary, - MemoryPool* pool = default_memory_pool()) + explicit DictionaryBuilderBase(const std::shared_ptr& dictionary, + MemoryPool* pool = default_memory_pool()) : ArrayBuilder(pool), memo_table_(new internal::DictionaryMemoTable(pool, dictionary)), delta_offset_(0), @@ -148,12 +190,18 @@ class DictionaryBuilderBase : public ArrayBuilder { /// \brief The current number of entries in the dictionary int64_t dictionary_length() const { return memo_table_->size(); } + /// \brief The value byte width (for FixedSizeBinaryType) + template + enable_if_fixed_size_binary byte_width() const { + return byte_width_; + } + /// \brief Append a scalar value - Status Append(const Scalar& value) { + Status Append(Value value) { ARROW_RETURN_NOT_OK(Reserve(1)); int32_t memo_index; - ARROW_RETURN_NOT_OK(memo_table_->GetOrInsert(value, &memo_index)); + ARROW_RETURN_NOT_OK(memo_table_->GetOrInsert(value, &memo_index)); ARROW_RETURN_NOT_OK(indices_builder_.Append(memo_index)); length_ += 1; @@ -190,6 +238,14 @@ class DictionaryBuilderBase : public ArrayBuilder { return Append(util::string_view(value, length)); } + /// \brief Append a decimal (only for Decimal128Type) + template + enable_if_decimal Append(const Decimal128& value) { + uint8_t data[16]; + value.ToBytes(data); + return Append(data, 16); + } + /// \brief Append a scalar null value Status AppendNull() final { length_ += 1; @@ -220,6 +276,11 @@ class DictionaryBuilderBase : public ArrayBuilder { const Array& array) { using ArrayType = typename TypeTraits::ArrayType; +#ifndef NDEBUG + ARROW_RETURN_NOT_OK(ArrayBuilder::CheckArrayType( + value_type_, array, "Wrong value type of array to be appended")); +#endif + const auto& concrete_array = static_cast(array); for (int64_t i = 0; i < array.length(); i++) { if (array.IsNull(i)) { @@ -233,10 +294,10 @@ class DictionaryBuilderBase : public ArrayBuilder { template enable_if_fixed_size_binary AppendArray(const Array& array) { - if (!value_type_->Equals(*array.type())) { - return Status::Invalid( - "Cannot append FixedSizeBinary array with non-matching type"); - } +#ifndef NDEBUG + ARROW_RETURN_NOT_OK(ArrayBuilder::CheckArrayType( + value_type_, array, "Wrong value type of array to be appended")); +#endif const auto& concrete_array = static_cast(array); for (int64_t i = 0; i < array.length(); i++) { @@ -335,15 +396,30 @@ class DictionaryBuilderBase : public ArrayBuilder { template class DictionaryBuilderBase : public ArrayBuilder { public: - DictionaryBuilderBase(const std::shared_ptr& value_type, - MemoryPool* pool = default_memory_pool()) + template + DictionaryBuilderBase( + enable_if_t::value, uint8_t> + start_int_size, + const std::shared_ptr& value_type, + MemoryPool* pool = default_memory_pool()) + : ArrayBuilder(pool), indices_builder_(start_int_size, pool) {} + + explicit DictionaryBuilderBase(const std::shared_ptr& value_type, + MemoryPool* pool = default_memory_pool()) : ArrayBuilder(pool), indices_builder_(pool) {} + template + explicit DictionaryBuilderBase( + enable_if_t::value, uint8_t> + start_int_size, + MemoryPool* pool = default_memory_pool()) + : ArrayBuilder(pool), indices_builder_(start_int_size, pool) {} + explicit DictionaryBuilderBase(MemoryPool* pool = default_memory_pool()) : ArrayBuilder(pool), indices_builder_(pool) {} - DictionaryBuilderBase(const std::shared_ptr& dictionary, - MemoryPool* pool = default_memory_pool()) + explicit DictionaryBuilderBase(const std::shared_ptr& dictionary, + MemoryPool* pool = default_memory_pool()) : ArrayBuilder(pool), indices_builder_(pool) {} /// \brief Append a scalar null value @@ -363,6 +439,10 @@ class DictionaryBuilderBase : public ArrayBuilder { /// \brief Append a whole dense array to the builder Status AppendArray(const Array& array) { +#ifndef NDEBUG + ARROW_RETURN_NOT_OK(ArrayBuilder::CheckArrayType( + Type::NA, array, "Wrong value type of array to be appended")); +#endif for (int64_t i = 0; i < array.length(); i++) { ARROW_RETURN_NOT_OK(AppendNull()); } diff --git a/cpp/src/arrow/array/builder_nested.cc b/cpp/src/arrow/array/builder_nested.cc index b8af62fab14e..c3786a8fab4b 100644 --- a/cpp/src/arrow/array/builder_nested.cc +++ b/cpp/src/arrow/array/builder_nested.cc @@ -54,6 +54,18 @@ MapBuilder::MapBuilder(MemoryPool* pool, const std::shared_ptr& ke : MapBuilder(pool, key_builder, item_builder, map(key_builder->type(), item_builder->type(), keys_sorted)) {} +MapBuilder::MapBuilder(MemoryPool* pool, + const std::shared_ptr& struct_builder, + const std::shared_ptr& type) + : ArrayBuilder(pool) { + auto map_type = internal::checked_cast(type.get()); + keys_sorted_ = map_type->keys_sorted(); + key_builder_ = struct_builder->child_builder(0); + item_builder_ = struct_builder->child_builder(1); + list_builder_ = + std::make_shared(pool, struct_builder, struct_builder->type()); +} + Status MapBuilder::Resize(int64_t capacity) { RETURN_NOT_OK(list_builder_->Resize(capacity)); capacity_ = list_builder_->capacity(); @@ -170,6 +182,19 @@ Status FixedSizeListBuilder::AppendNulls(int64_t length) { return value_builder_->AppendNulls(list_size_ * length); } +Status FixedSizeListBuilder::ValidateOverflow(int64_t new_elements) { + auto new_length = value_builder_->length() + new_elements; + if (new_elements != list_size_) { + return Status::Invalid("Length of item not correct: expected ", list_size_, + " but got array of size ", new_elements); + } + if (new_length > maximum_elements()) { + return Status::CapacityError("array cannot contain more than ", maximum_elements(), + " elements, have ", new_elements); + } + return Status::OK(); +} + Status FixedSizeListBuilder::Resize(int64_t capacity) { RETURN_NOT_OK(CheckCapacity(capacity)); return ArrayBuilder::Resize(capacity); diff --git a/cpp/src/arrow/array/builder_nested.h b/cpp/src/arrow/array/builder_nested.h index cd6fadfcc2fe..b8948403acca 100644 --- a/cpp/src/arrow/array/builder_nested.h +++ b/cpp/src/arrow/array/builder_nested.h @@ -100,7 +100,7 @@ class BaseListBuilder : public ArrayBuilder { Status AppendNulls(int64_t length) final { ARROW_RETURN_NOT_OK(Reserve(length)); - ARROW_RETURN_NOT_OK(CheckNextOffset()); + ARROW_RETURN_NOT_OK(ValidateOverflow(0)); UnsafeAppendToBitmap(length, false); const int64_t num_values = value_builder_->length(); for (int64_t i = 0; i < length; ++i) { @@ -131,6 +131,16 @@ class BaseListBuilder : public ArrayBuilder { return Status::OK(); } + Status ValidateOverflow(int64_t new_elements) const { + auto new_length = value_builder_->length() + new_elements; + if (ARROW_PREDICT_FALSE(new_length > maximum_elements())) { + return Status::CapacityError("List array cannot contain more than ", + maximum_elements(), " elements, have ", new_elements); + } else { + return Status::OK(); + } + } + ArrayBuilder* value_builder() const { return value_builder_.get(); } // Cannot make this a static attribute because of linking issues @@ -147,17 +157,8 @@ class BaseListBuilder : public ArrayBuilder { std::shared_ptr value_builder_; std::shared_ptr value_field_; - Status CheckNextOffset() const { - const int64_t num_values = value_builder_->length(); - ARROW_RETURN_IF( - num_values > maximum_elements(), - Status::CapacityError("List array cannot contain more than ", maximum_elements(), - " child elements,", " have ", num_values)); - return Status::OK(); - } - Status AppendNextOffset() { - ARROW_RETURN_NOT_OK(CheckNextOffset()); + ARROW_RETURN_NOT_OK(ValidateOverflow(0)); const int64_t num_values = value_builder_->length(); return offsets_builder_.Append(static_cast(num_values)); } @@ -227,6 +228,9 @@ class ARROW_EXPORT MapBuilder : public ArrayBuilder { MapBuilder(MemoryPool* pool, const std::shared_ptr& key_builder, const std::shared_ptr& item_builder, bool keys_sorted = false); + MapBuilder(MemoryPool* pool, const std::shared_ptr& item_builder, + const std::shared_ptr& type); + Status Resize(int64_t capacity) override; void Reset() override; Status FinishInternal(std::shared_ptr* out) override; @@ -276,6 +280,10 @@ class ARROW_EXPORT MapBuilder : public ArrayBuilder { return map(key_builder_->type(), item_builder_->type(), keys_sorted_); } + Status ValidateOverflow(int64_t new_elements) { + return list_builder_->ValidateOverflow(new_elements); + } + protected: inline Status AdjustStructBuilderLength(); @@ -343,12 +351,19 @@ class ARROW_EXPORT FixedSizeListBuilder : public ArrayBuilder { /// automatically. Status AppendNulls(int64_t length) final; + Status ValidateOverflow(int64_t new_elements); + ArrayBuilder* value_builder() const { return value_builder_.get(); } std::shared_ptr type() const override { return fixed_size_list(value_field_->WithType(value_builder_->type()), list_size_); } + // Cannot make this a static attribute because of linking issues + static constexpr int64_t maximum_elements() { + return std::numeric_limits::max() - 1; + } + protected: std::shared_ptr value_field_; const int32_t list_size_; diff --git a/cpp/src/arrow/array/builder_primitive.h b/cpp/src/arrow/array/builder_primitive.h index e6b1baa5879f..cc907fb6b8a9 100644 --- a/cpp/src/arrow/array/builder_primitive.h +++ b/cpp/src/arrow/array/builder_primitive.h @@ -31,6 +31,9 @@ namespace arrow { class ARROW_EXPORT NullBuilder : public ArrayBuilder { public: explicit NullBuilder(MemoryPool* pool = default_memory_pool()) : ArrayBuilder(pool) {} + explicit NullBuilder(const std::shared_ptr& type, + MemoryPool* pool = default_memory_pool()) + : NullBuilder(pool) {} /// \brief Append the specified number of null elements Status AppendNulls(int64_t length) final { diff --git a/cpp/src/arrow/array/concatenate.cc b/cpp/src/arrow/array/concatenate.cc index 923bf308bc50..dcfb3f530042 100644 --- a/cpp/src/arrow/array/concatenate.cc +++ b/cpp/src/arrow/array/concatenate.cc @@ -36,12 +36,14 @@ #include "arrow/util/bit_util.h" #include "arrow/util/bitmap_ops.h" #include "arrow/util/checked_cast.h" -#include "arrow/util/int_util.h" +#include "arrow/util/int_util_internal.h" #include "arrow/util/logging.h" #include "arrow/visitor_inline.h" namespace arrow { +using internal::SafeSignedAdd; + /// offset, length pair for representing a Range of a buffer or array struct Range { int64_t offset = -1, length = 0; @@ -68,7 +70,9 @@ static Status ConcatenateBitmaps(const std::vector& bitmaps, MemoryPool* std::shared_ptr* out) { int64_t out_length = 0; for (const auto& bitmap : bitmaps) { - out_length += bitmap.range.length; + if (internal::AddWithOverflow(out_length, bitmap.range.length, &out_length)) { + return Status::Invalid("Length overflow when concatenating arrays"); + } } ARROW_ASSIGN_OR_RAISE(*out, AllocateBitmap(out_length, pool)); uint8_t* dst = (*out)->mutable_data(); @@ -84,10 +88,6 @@ static Status ConcatenateBitmaps(const std::vector& bitmaps, MemoryPool* bitmap_offset += bitmap.range.length; } - // finally (if applicable) zero out any trailing bits - if (auto preceding_bits = BitUtil::kPrecedingBitmask[out_length % 8]) { - dst[out_length / 8] &= preceding_bits; - } return Status::OK(); } @@ -158,7 +158,7 @@ static Status PutOffsets(const std::shared_ptr& src, Offset first_offset // Avoid UB on non-validated input by doing the addition in the unsigned domain. // (the result can later be validated using Array::ValidateFull) std::transform(src_begin, src_end, dst, [adjustment](Offset offset) { - return internal::SafeSignedAdd(offset, adjustment); + return SafeSignedAdd(offset, adjustment); }); return Status::OK(); } @@ -170,13 +170,13 @@ class ConcatenateImpl { : in_(std::move(in)), pool_(pool), out_(std::make_shared()) { out_->type = in[0]->type; for (size_t i = 0; i < in_.size(); ++i) { - out_->length += in[i]->length; + out_->length = SafeSignedAdd(out_->length, in[i]->length); if (out_->null_count == kUnknownNullCount || in[i]->null_count == kUnknownNullCount) { out_->null_count = kUnknownNullCount; continue; } - out_->null_count += in[i]->null_count; + out_->null_count = SafeSignedAdd(out_->null_count.load(), in[i]->null_count.load()); } out_->buffers.resize(in[0]->buffers.size()); out_->child_data.resize(in[0]->child_data.size()); diff --git a/cpp/src/arrow/array/data.cc b/cpp/src/arrow/array/data.cc index 6af1c443ad93..9c5e630bb2bd 100644 --- a/cpp/src/arrow/array/data.cc +++ b/cpp/src/arrow/array/data.cc @@ -29,7 +29,7 @@ #include "arrow/status.h" #include "arrow/type.h" #include "arrow/util/bitmap_ops.h" -#include "arrow/util/int_util.h" +#include "arrow/util/int_util_internal.h" #include "arrow/util/logging.h" #include "arrow/util/macros.h" @@ -100,6 +100,8 @@ std::shared_ptr ArrayData::Slice(int64_t off, int64_t len) const { copy->offset = off; if (null_count == length) { copy->null_count = len; + } else if (off == offset && len == length) { // A copy of current. + copy->null_count = null_count.load(); } else { copy->null_count = null_count != 0 ? kUnknownNullCount : 0; } diff --git a/cpp/src/arrow/array/data.h b/cpp/src/arrow/array/data.h index 536a6c2a5f41..02a49949e1f6 100644 --- a/cpp/src/arrow/array/data.h +++ b/cpp/src/arrow/array/data.h @@ -180,6 +180,22 @@ struct ARROW_EXPORT ArrayData { return GetValues(i, offset); } + // Like GetValues, but returns NULLPTR instead of aborting if the underlying + // buffer is not a CPU buffer. + template + inline const T* GetValuesSafe(int i, int64_t absolute_offset) const { + if (buffers[i] && buffers[i]->is_cpu()) { + return reinterpret_cast(buffers[i]->data()) + absolute_offset; + } else { + return NULLPTR; + } + } + + template + inline const T* GetValuesSafe(int i) const { + return GetValuesSafe(i, offset); + } + // Access a buffer's data as a typed C pointer template inline T* GetMutableValues(int i, int64_t absolute_offset) { diff --git a/cpp/src/arrow/array/dict_internal.h b/cpp/src/arrow/array/dict_internal.h index 5bf584c82162..aa027ac22dea 100644 --- a/cpp/src/arrow/array/dict_internal.h +++ b/cpp/src/arrow/array/dict_internal.h @@ -189,16 +189,5 @@ struct DictionaryTraits> { } }; -template -struct DictionaryCTraits { - using ArrowType = typename CTypeTraits::ArrowType; - using MemoTableType = typename DictionaryTraits::MemoTableType; -}; - -template <> -struct DictionaryCTraits { - using MemoTableType = DictionaryTraits::MemoTableType; -}; - } // namespace internal } // namespace arrow diff --git a/cpp/src/arrow/array/validate.cc b/cpp/src/arrow/array/validate.cc index 8fb8b59b9bcc..3063f5580cda 100644 --- a/cpp/src/arrow/array/validate.cc +++ b/cpp/src/arrow/array/validate.cc @@ -26,7 +26,7 @@ #include "arrow/type_traits.h" #include "arrow/util/bit_util.h" #include "arrow/util/checked_cast.h" -#include "arrow/util/int_util.h" +#include "arrow/util/int_util_internal.h" #include "arrow/util/logging.h" #include "arrow/visitor_inline.h" @@ -46,9 +46,6 @@ struct ValidateArrayVisitor { } Status Visit(const PrimitiveArray& array) { - ARROW_RETURN_IF(array.data()->buffers.size() != 2, - Status::Invalid("number of buffers is != 2")); - if (array.length() > 0) { if (array.data()->buffers[1] == nullptr) { return Status::Invalid("values buffer is null"); @@ -61,9 +58,6 @@ struct ValidateArrayVisitor { } Status Visit(const Decimal128Array& array) { - if (array.data()->buffers.size() != 2) { - return Status::Invalid("number of buffers is != 2"); - } if (array.length() > 0 && array.values() == nullptr) { return Status::Invalid("values is null"); } @@ -98,8 +92,9 @@ struct ValidateArrayVisitor { if (value_size < 0) { return Status::Invalid("FixedSizeListArray has negative value size ", value_size); } - if (HasPositiveMultiplyOverflow(len, value_size) || - array.values()->length() != len * value_size) { + int64_t expected_values_length = -1; + if (MultiplyWithOverflow(len, value_size, &expected_values_length) || + array.values()->length() != expected_values_length) { return Status::Invalid("Values Length (", array.values()->length(), ") is not equal to the length (", len, ") multiplied by the value size (", value_size, ")"); @@ -211,15 +206,12 @@ struct ValidateArrayVisitor { protected: template Status ValidateBinaryArray(const BinaryArrayType& array) { - if (array.data()->buffers.size() != 3) { - return Status::Invalid("number of buffers is != 3"); - } if (array.value_data() == nullptr) { return Status::Invalid("value data buffer is null"); } RETURN_NOT_OK(ValidateOffsets(array)); - if (array.length() > 0) { + if (array.length() > 0 && array.value_offsets()->is_cpu()) { const auto first_offset = array.value_offset(0); const auto last_offset = array.value_offset(array.length()); // This early test avoids undefined behaviour when computing `data_extent` @@ -250,7 +242,7 @@ struct ValidateArrayVisitor { RETURN_NOT_OK(ValidateOffsets(array)); // An empty list array can have 0 offsets - if (array.length() > 0) { + if (array.length() > 0 && array.value_offsets()->is_cpu()) { const auto first_offset = array.value_offset(0); const auto last_offset = array.value_offset(array.length()); // This early test avoids undefined behaviour when computing `data_extent` @@ -329,7 +321,8 @@ Status ValidateArray(const Array& array) { type.ToString(), ", got ", data.buffers.size()); } // This check is required to avoid addition overflow below - if (HasPositiveAdditionOverflow(array.length(), array.offset())) { + int64_t length_plus_offset = -1; + if (AddWithOverflow(array.length(), array.offset(), &length_plus_offset)) { return Status::Invalid("Array of type ", type.ToString(), " has impossibly large length and offset"); } @@ -343,15 +336,13 @@ Status ValidateArray(const Array& array) { int64_t min_buffer_size = -1; switch (spec.kind) { case DataTypeLayout::BITMAP: - min_buffer_size = BitUtil::BytesForBits(array.length() + array.offset()); + min_buffer_size = BitUtil::BytesForBits(length_plus_offset); break; case DataTypeLayout::FIXED_WIDTH: - if (HasPositiveMultiplyOverflow(array.length() + array.offset(), - spec.byte_width)) { + if (MultiplyWithOverflow(length_plus_offset, spec.byte_width, &min_buffer_size)) { return Status::Invalid("Array of type ", type.ToString(), " has impossibly large length and offset"); } - min_buffer_size = spec.byte_width * (array.length() + array.offset()); break; case DataTypeLayout::ALWAYS_NULL: // XXX Should we raise on non-null buffer? diff --git a/cpp/src/arrow/arrow.pc.in b/cpp/src/arrow/arrow.pc.in index b3d32ffe4b52..947d534fdbfe 100644 --- a/cpp/src/arrow/arrow.pc.in +++ b/cpp/src/arrow/arrow.pc.in @@ -26,4 +26,4 @@ Name: Apache Arrow Description: Arrow is a set of technologies that enable big-data systems to process and move data fast. Version: @ARROW_VERSION@ Libs: -L${libdir} -larrow -Cflags: -I${includedir} @ARROW_DEFINITION_FLAGS@ +Cflags: -I${includedir} diff --git a/cpp/src/arrow/buffer.cc b/cpp/src/arrow/buffer.cc index 2614cd17f8a3..9215d9ab544d 100644 --- a/cpp/src/arrow/buffer.cc +++ b/cpp/src/arrow/buffer.cc @@ -25,7 +25,7 @@ #include "arrow/result.h" #include "arrow/status.h" #include "arrow/util/bit_util.h" -#include "arrow/util/int_util.h" +#include "arrow/util/int_util_internal.h" #include "arrow/util/logging.h" #include "arrow/util/string.h" @@ -206,7 +206,7 @@ class PoolBuffer : public ResizableBuffer { } Status Resize(const int64_t new_size, bool shrink_to_fit = true) override { - if (new_size < 0) { + if (ARROW_PREDICT_FALSE(new_size < 0)) { return Status::Invalid("Negative buffer resize: ", new_size); } if (mutable_data_ && shrink_to_fit && new_size <= size_) { @@ -277,13 +277,18 @@ Result> AllocateResizableBuffer(const int64_t s } Result> AllocateBitmap(int64_t length, MemoryPool* pool) { - return AllocateBuffer(BitUtil::BytesForBits(length), pool); + ARROW_ASSIGN_OR_RAISE(auto buf, AllocateBuffer(BitUtil::BytesForBits(length), pool)); + // Zero out any trailing bits + if (buf->size() > 0) { + buf->mutable_data()[buf->size() - 1] = 0; + } + return std::move(buf); } Result> AllocateEmptyBitmap(int64_t length, MemoryPool* pool) { - ARROW_ASSIGN_OR_RAISE(auto buf, AllocateBitmap(length, pool)); + ARROW_ASSIGN_OR_RAISE(auto buf, AllocateBuffer(BitUtil::BytesForBits(length), pool)); memset(buf->mutable_data(), 0, static_cast(buf->size())); - return buf; + return std::move(buf); } Status AllocateEmptyBitmap(int64_t length, std::shared_ptr* out) { diff --git a/cpp/src/arrow/builder.cc b/cpp/src/arrow/builder.cc index 6be07d6ca75b..1dcbf7851abd 100644 --- a/cpp/src/arrow/builder.cc +++ b/cpp/src/arrow/builder.cc @@ -40,9 +40,17 @@ struct DictionaryBuilderCase { return CreateFor(); } + Status Visit(const NullType&) { return CreateFor(); } Status Visit(const BinaryType&) { return Create(); } Status Visit(const StringType&) { return Create(); } + Status Visit(const LargeBinaryType&) { + return Create>(); + } + Status Visit(const LargeStringType&) { + return Create>(); + } Status Visit(const FixedSizeBinaryType&) { return CreateFor(); } + Status Visit(const Decimal128Type&) { return CreateFor(); } Status Visit(const DataType& value_type) { return NotImplemented(value_type); } Status Visit(const HalfFloatType& value_type) { return NotImplemented(value_type); } @@ -59,17 +67,21 @@ struct DictionaryBuilderCase { template Status Create() { + BuilderType* builder; if (dictionary != nullptr) { - out->reset(new BuilderType(dictionary, pool)); + builder = new BuilderType(dictionary, pool); } else { - out->reset(new BuilderType(value_type, pool)); + auto start_int_size = internal::GetByteWidth(*index_type); + builder = new BuilderType(start_int_size, value_type, pool); } + out->reset(builder); return Status::OK(); } Status Make() { return VisitTypeInline(*value_type, this); } MemoryPool* pool; + const std::shared_ptr& index_type; const std::shared_ptr& value_type; const std::shared_ptr& dictionary; std::unique_ptr* out; @@ -129,7 +141,8 @@ Status MakeBuilder(MemoryPool* pool, const std::shared_ptr& type, case Type::DICTIONARY: { const auto& dict_type = static_cast(*type); - DictionaryBuilderCase visitor = {pool, dict_type.value_type(), nullptr, out}; + DictionaryBuilderCase visitor = {pool, dict_type.index_type(), + dict_type.value_type(), nullptr, out}; return visitor.Make(); } @@ -199,7 +212,8 @@ Status MakeDictionaryBuilder(MemoryPool* pool, const std::shared_ptr& const std::shared_ptr& dictionary, std::unique_ptr* out) { const auto& dict_type = static_cast(*type); - DictionaryBuilderCase visitor = {pool, dict_type.value_type(), dictionary, out}; + DictionaryBuilderCase visitor = {pool, dict_type.index_type(), dict_type.value_type(), + dictionary, out}; return visitor.Make(); } diff --git a/cpp/src/arrow/builder.h b/cpp/src/arrow/builder.h index 3202312c47e6..54ff2904b91d 100644 --- a/cpp/src/arrow/builder.h +++ b/cpp/src/arrow/builder.h @@ -39,7 +39,7 @@ class MemoryPool; /// \brief Construct an empty ArrayBuilder corresponding to the data /// type /// \param[in] pool the MemoryPool to use for allocations -/// \param[in] type an instance of DictionaryType +/// \param[in] type the data type to create the builder for /// \param[out] out the created ArrayBuilder ARROW_EXPORT Status MakeBuilder(MemoryPool* pool, const std::shared_ptr& type, @@ -48,7 +48,7 @@ Status MakeBuilder(MemoryPool* pool, const std::shared_ptr& type, /// \brief Construct an empty DictionaryBuilder initialized optionally /// with a pre-existing dictionary /// \param[in] pool the MemoryPool to use for allocations -/// \param[in] type an instance of DictionaryType +/// \param[in] type the dictionary type to create the builder for /// \param[in] dictionary the initial dictionary, if any. May be nullptr /// \param[out] out the created ArrayBuilder ARROW_EXPORT diff --git a/cpp/src/arrow/c/abi.h b/cpp/src/arrow/c/abi.h index 821bc961281c..a78170dbdbcb 100644 --- a/cpp/src/arrow/c/abi.h +++ b/cpp/src/arrow/c/abi.h @@ -60,6 +60,44 @@ struct ArrowArray { void* private_data; }; +// EXPERIMENTAL: C stream interface + +struct ArrowArrayStream { + // Callback to get the stream type + // (will be the same for all arrays in the stream). + // + // Return value: 0 if successful, an `errno`-compatible error code otherwise. + // + // If successful, the ArrowSchema must be released independently from the stream. + int (*get_schema)(struct ArrowArrayStream*, struct ArrowSchema* out); + + // Callback to get the next array + // (if no error and the array is released, the stream has ended) + // + // Return value: 0 if successful, an `errno`-compatible error code otherwise. + // + // If successful, the ArrowArray must be released independently from the stream. + int (*get_next)(struct ArrowArrayStream*, struct ArrowArray* out); + + // Callback to get optional detailed error information. + // This must only be called if the last stream operation failed + // with a non-0 return code. + // + // Return value: pointer to a null-terminated character array describing + // the last error, or NULL if no description is available. + // + // The returned pointer is only valid until the next operation on this stream + // (including release). + const char* (*get_last_error)(struct ArrowArrayStream*); + + // Release callback: release the stream's own resources. + // Note that arrays returned by `get_next` must be individually released. + void (*release)(struct ArrowArrayStream*); + + // Opaque producer-specific data + void* private_data; +}; + #ifdef __cplusplus } #endif diff --git a/cpp/src/arrow/c/bridge.cc b/cpp/src/arrow/c/bridge.cc index 1e602a6a310f..5b360abc48c0 100644 --- a/cpp/src/arrow/c/bridge.cc +++ b/cpp/src/arrow/c/bridge.cc @@ -18,6 +18,7 @@ #include "arrow/c/bridge.h" #include +#include #include #include #include @@ -1501,4 +1502,197 @@ Result> ImportRecordBatch(struct ArrowArray* array, return ImportRecordBatch(array, *maybe_schema); } +////////////////////////////////////////////////////////////////////////// +// C stream export + +namespace { + +class ExportedArrayStream { + public: + struct PrivateData { + explicit PrivateData(std::shared_ptr reader) + : reader_(std::move(reader)) {} + + std::shared_ptr reader_; + std::string last_error_; + + PrivateData() = default; + ARROW_DISALLOW_COPY_AND_ASSIGN(PrivateData); + }; + + explicit ExportedArrayStream(struct ArrowArrayStream* stream) : stream_(stream) {} + + Status GetSchema(struct ArrowSchema* out_schema) { + return ExportSchema(*reader()->schema(), out_schema); + } + + Status GetNext(struct ArrowArray* out_array) { + std::shared_ptr batch; + RETURN_NOT_OK(reader()->ReadNext(&batch)); + if (batch == nullptr) { + // End of stream + ArrowArrayMarkReleased(out_array); + return Status::OK(); + } else { + return ExportRecordBatch(*batch, out_array); + } + } + + const char* GetLastError() { + const auto& last_error = private_data()->last_error_; + return last_error.empty() ? nullptr : last_error.c_str(); + } + + void Release() { + if (ArrowArrayStreamIsReleased(stream_)) { + return; + } + DCHECK_NE(private_data(), nullptr); + delete private_data(); + + ArrowArrayStreamMarkReleased(stream_); + } + + // C-compatible callbacks + + static int StaticGetSchema(struct ArrowArrayStream* stream, + struct ArrowSchema* out_schema) { + ExportedArrayStream self{stream}; + return self.ToCError(self.GetSchema(out_schema)); + } + + static int StaticGetNext(struct ArrowArrayStream* stream, + struct ArrowArray* out_array) { + ExportedArrayStream self{stream}; + return self.ToCError(self.GetNext(out_array)); + } + + static void StaticRelease(struct ArrowArrayStream* stream) { + ExportedArrayStream{stream}.Release(); + } + + static const char* StaticGetLastError(struct ArrowArrayStream* stream) { + return ExportedArrayStream{stream}.GetLastError(); + } + + private: + int ToCError(const Status& status) { + if (ARROW_PREDICT_TRUE(status.ok())) { + private_data()->last_error_.clear(); + return 0; + } + private_data()->last_error_ = status.ToString(); + switch (status.code()) { + case StatusCode::IOError: + return EIO; + case StatusCode::NotImplemented: + return ENOSYS; + case StatusCode::OutOfMemory: + return ENOMEM; + default: + return EINVAL; // Fallback for Invalid, TypeError, etc. + } + } + + PrivateData* private_data() { + return reinterpret_cast(stream_->private_data); + } + + const std::shared_ptr& reader() { return private_data()->reader_; } + + struct ArrowArrayStream* stream_; +}; + +} // namespace + +Status ExportRecordBatchReader(std::shared_ptr reader, + struct ArrowArrayStream* out) { + out->get_schema = ExportedArrayStream::StaticGetSchema; + out->get_next = ExportedArrayStream::StaticGetNext; + out->get_last_error = ExportedArrayStream::StaticGetLastError; + out->release = ExportedArrayStream::StaticRelease; + out->private_data = new ExportedArrayStream::PrivateData{std::move(reader)}; + return Status::OK(); +} + +////////////////////////////////////////////////////////////////////////// +// C stream import + +namespace { + +class ArrayStreamBatchReader : public RecordBatchReader { + public: + explicit ArrayStreamBatchReader(struct ArrowArrayStream* stream) { + ArrowArrayStreamMove(stream, &stream_); + DCHECK(!ArrowArrayStreamIsReleased(&stream_)); + } + + ~ArrayStreamBatchReader() { + ArrowArrayStreamRelease(&stream_); + DCHECK(ArrowArrayStreamIsReleased(&stream_)); + } + + std::shared_ptr schema() const override { return CacheSchema(); } + + Status ReadNext(std::shared_ptr* batch) override { + struct ArrowArray c_array; + RETURN_NOT_OK(StatusFromCError(stream_.get_next(&stream_, &c_array))); + if (ArrowArrayIsReleased(&c_array)) { + // End of stream + batch->reset(); + return Status::OK(); + } else { + return ImportRecordBatch(&c_array, CacheSchema()).Value(batch); + } + } + + private: + std::shared_ptr CacheSchema() const { + if (!schema_) { + struct ArrowSchema c_schema; + ARROW_CHECK_OK(StatusFromCError(stream_.get_schema(&stream_, &c_schema))); + schema_ = ImportSchema(&c_schema).ValueOrDie(); + } + return schema_; + } + + Status StatusFromCError(int errno_like) const { + if (ARROW_PREDICT_TRUE(errno_like == 0)) { + return Status::OK(); + } + StatusCode code; + switch (errno_like) { + case EDOM: + case EINVAL: + case ERANGE: + code = StatusCode::Invalid; + break; + case ENOMEM: + code = StatusCode::OutOfMemory; + break; + case ENOSYS: + code = StatusCode::NotImplemented; + default: + code = StatusCode::IOError; + break; + } + const char* last_error = stream_.get_last_error(&stream_); + return Status(code, last_error ? std::string(last_error) : ""); + } + + mutable struct ArrowArrayStream stream_; + mutable std::shared_ptr schema_; +}; + +} // namespace + +Result> ImportRecordBatchReader( + struct ArrowArrayStream* stream) { + if (ArrowArrayStreamIsReleased(stream)) { + return Status::Invalid("Cannot import released ArrowArrayStream"); + } + // XXX should we call get_schema() here to avoid crashing on error? + return std::make_shared(stream); +} + } // namespace arrow diff --git a/cpp/src/arrow/c/bridge.h b/cpp/src/arrow/c/bridge.h index 8efb5d98bed0..294f53e49fb5 100644 --- a/cpp/src/arrow/c/bridge.h +++ b/cpp/src/arrow/c/bridge.h @@ -29,6 +29,10 @@ namespace arrow { +/// \defgroup c-data-interface Functions for working with the C data interface. +/// +/// @{ + /// \brief Export C++ DataType using the C data interface format. /// /// The root type is considered to have empty name and metadata. @@ -160,4 +164,34 @@ ARROW_EXPORT Result> ImportRecordBatch(struct ArrowArray* array, struct ArrowSchema* schema); +/// @} + +/// \defgroup c-stream-interface Functions for working with the C data interface. +/// +/// @{ + +/// \brief EXPERIMENTAL: Export C++ RecordBatchReader using the C stream interface. +/// +/// The resulting ArrowArrayStream struct keeps the record batch reader alive +/// until its release callback is called by the consumer. +/// +/// \param[in] reader RecordBatchReader object to export +/// \param[out] out C struct where to export the stream +ARROW_EXPORT +Status ExportRecordBatchReader(std::shared_ptr reader, + struct ArrowArrayStream* out); + +/// \brief EXPERIMENTAL: Import C++ RecordBatchReader from the C stream interface. +/// +/// The ArrowArrayStream struct has its contents moved to a private object +/// held alive by the resulting record batch reader. +/// +/// \param[in,out] stream C stream interface struct +/// \return Imported RecordBatchReader object +ARROW_EXPORT +Result> ImportRecordBatchReader( + struct ArrowArrayStream* stream); + +/// @} + } // namespace arrow diff --git a/cpp/src/arrow/c/bridge_test.cc b/cpp/src/arrow/c/bridge_test.cc index 6695d6ed5db2..3f84edfc2a60 100644 --- a/cpp/src/arrow/c/bridge_test.cc +++ b/cpp/src/arrow/c/bridge_test.cc @@ -15,6 +15,7 @@ // specific language governing permissions and limitations // under the License. +#include #include #include #include @@ -22,6 +23,7 @@ #include #include +#include #include #include "arrow/c/bridge.h" @@ -40,6 +42,8 @@ namespace arrow { using internal::ArrayExportGuard; using internal::ArrayExportTraits; +using internal::ArrayStreamExportGuard; +using internal::ArrayStreamExportTraits; using internal::SchemaExportGuard; using internal::SchemaExportTraits; @@ -78,11 +82,11 @@ class ReleaseCallback { explicit ReleaseCallback(CType* c_struct) : called_(false) { orig_release_ = c_struct->release; orig_private_data_ = c_struct->private_data; - c_struct->release = ReleaseUnbound; + c_struct->release = StaticRelease; c_struct->private_data = this; } - static void ReleaseUnbound(CType* c_struct) { + static void StaticRelease(CType* c_struct) { reinterpret_cast(c_struct->private_data)->Release(c_struct); } @@ -2678,4 +2682,248 @@ TEST_F(TestArrayRoundtrip, RecordBatch) { // TODO C -> C++ -> C roundtripping tests? +//////////////////////////////////////////////////////////////////////////// +// Array stream export tests + +class FailingRecordBatchReader : public RecordBatchReader { + public: + explicit FailingRecordBatchReader(Status error) : error_(std::move(error)) {} + + static std::shared_ptr expected_schema() { return arrow::schema({}); } + + std::shared_ptr schema() const override { return expected_schema(); } + + Status ReadNext(std::shared_ptr* batch) override { return error_; } + + protected: + Status error_; +}; + +class BaseArrayStreamTest : public ::testing::Test { + public: + void SetUp() override { + pool_ = default_memory_pool(); + orig_allocated_ = pool_->bytes_allocated(); + } + + void TearDown() override { ASSERT_EQ(pool_->bytes_allocated(), orig_allocated_); } + + RecordBatchVector MakeBatches(std::shared_ptr schema, ArrayVector arrays) { + DCHECK_EQ(schema->num_fields(), 1); + RecordBatchVector batches; + for (const auto& array : arrays) { + batches.push_back(RecordBatch::Make(schema, array->length(), {array})); + } + return batches; + } + + protected: + MemoryPool* pool_; + int64_t orig_allocated_; +}; + +class TestArrayStreamExport : public BaseArrayStreamTest { + public: + void AssertStreamSchema(struct ArrowArrayStream* c_stream, const Schema& expected) { + struct ArrowSchema c_schema; + ASSERT_EQ(0, c_stream->get_schema(c_stream, &c_schema)); + + SchemaExportGuard schema_guard(&c_schema); + ASSERT_FALSE(ArrowSchemaIsReleased(&c_schema)); + ASSERT_OK_AND_ASSIGN(auto schema, ImportSchema(&c_schema)); + AssertSchemaEqual(expected, *schema); + } + + void AssertStreamEnd(struct ArrowArrayStream* c_stream) { + struct ArrowArray c_array; + ASSERT_EQ(0, c_stream->get_next(c_stream, &c_array)); + + ArrayExportGuard guard(&c_array); + ASSERT_TRUE(ArrowArrayIsReleased(&c_array)); + } + + void AssertStreamNext(struct ArrowArrayStream* c_stream, const RecordBatch& expected) { + struct ArrowArray c_array; + ASSERT_EQ(0, c_stream->get_next(c_stream, &c_array)); + + ArrayExportGuard guard(&c_array); + ASSERT_FALSE(ArrowArrayIsReleased(&c_array)); + + ASSERT_OK_AND_ASSIGN(auto batch, ImportRecordBatch(&c_array, expected.schema())); + AssertBatchesEqual(expected, *batch); + } +}; + +TEST_F(TestArrayStreamExport, Empty) { + auto schema = arrow::schema({field("ints", int32())}); + auto batches = MakeBatches(schema, {}); + ASSERT_OK_AND_ASSIGN(auto reader, RecordBatchReader::Make(batches, schema)); + + struct ArrowArrayStream c_stream; + + ASSERT_OK(ExportRecordBatchReader(reader, &c_stream)); + ArrayStreamExportGuard guard(&c_stream); + + ASSERT_FALSE(ArrowArrayStreamIsReleased(&c_stream)); + AssertStreamSchema(&c_stream, *schema); + AssertStreamEnd(&c_stream); + AssertStreamEnd(&c_stream); +} + +TEST_F(TestArrayStreamExport, Simple) { + auto schema = arrow::schema({field("ints", int32())}); + auto batches = MakeBatches( + schema, {ArrayFromJSON(int32(), "[1, 2]"), ArrayFromJSON(int32(), "[4, 5, null]")}); + ASSERT_OK_AND_ASSIGN(auto reader, RecordBatchReader::Make(batches, schema)); + + struct ArrowArrayStream c_stream; + + ASSERT_OK(ExportRecordBatchReader(reader, &c_stream)); + ArrayStreamExportGuard guard(&c_stream); + + ASSERT_FALSE(ArrowArrayStreamIsReleased(&c_stream)); + AssertStreamSchema(&c_stream, *schema); + AssertStreamNext(&c_stream, *batches[0]); + AssertStreamNext(&c_stream, *batches[1]); + AssertStreamEnd(&c_stream); + AssertStreamEnd(&c_stream); +} + +TEST_F(TestArrayStreamExport, ArrayLifetime) { + auto schema = arrow::schema({field("ints", int32())}); + auto batches = MakeBatches( + schema, {ArrayFromJSON(int32(), "[1, 2]"), ArrayFromJSON(int32(), "[4, 5, null]")}); + ASSERT_OK_AND_ASSIGN(auto reader, RecordBatchReader::Make(batches, schema)); + + struct ArrowArrayStream c_stream; + struct ArrowSchema c_schema; + struct ArrowArray c_array0, c_array1; + + ASSERT_OK(ExportRecordBatchReader(reader, &c_stream)); + { + ArrayStreamExportGuard guard(&c_stream); + ASSERT_FALSE(ArrowArrayStreamIsReleased(&c_stream)); + + ASSERT_EQ(0, c_stream.get_schema(&c_stream, &c_schema)); + ASSERT_EQ(0, c_stream.get_next(&c_stream, &c_array0)); + ASSERT_EQ(0, c_stream.get_next(&c_stream, &c_array1)); + AssertStreamEnd(&c_stream); + } + + ArrayExportGuard guard0(&c_array0), guard1(&c_array1); + + { + SchemaExportGuard schema_guard(&c_schema); + ASSERT_OK_AND_ASSIGN(auto got_schema, ImportSchema(&c_schema)); + AssertSchemaEqual(*schema, *got_schema); + } + + ASSERT_GT(pool_->bytes_allocated(), orig_allocated_); + ASSERT_OK_AND_ASSIGN(auto batch, ImportRecordBatch(&c_array1, schema)); + AssertBatchesEqual(*batches[1], *batch); + ASSERT_OK_AND_ASSIGN(batch, ImportRecordBatch(&c_array0, schema)); + AssertBatchesEqual(*batches[0], *batch); +} + +TEST_F(TestArrayStreamExport, Errors) { + auto reader = + std::make_shared(Status::Invalid("some example error")); + + struct ArrowArrayStream c_stream; + + ASSERT_OK(ExportRecordBatchReader(reader, &c_stream)); + ArrayStreamExportGuard guard(&c_stream); + + struct ArrowSchema c_schema; + ASSERT_EQ(0, c_stream.get_schema(&c_stream, &c_schema)); + ASSERT_FALSE(ArrowSchemaIsReleased(&c_schema)); + { + SchemaExportGuard schema_guard(&c_schema); + ASSERT_OK_AND_ASSIGN(auto schema, ImportSchema(&c_schema)); + AssertSchemaEqual(schema, arrow::schema({})); + } + + struct ArrowArray c_array; + ASSERT_EQ(EINVAL, c_stream.get_next(&c_stream, &c_array)); +} + +//////////////////////////////////////////////////////////////////////////// +// Array stream roundtrip tests + +class TestArrayStreamRoundtrip : public BaseArrayStreamTest { + public: + void Roundtrip(std::shared_ptr* reader, + struct ArrowArrayStream* c_stream) { + ASSERT_OK(ExportRecordBatchReader(*reader, c_stream)); + ASSERT_FALSE(ArrowArrayStreamIsReleased(c_stream)); + + ASSERT_OK_AND_ASSIGN(auto got_reader, ImportRecordBatchReader(c_stream)); + *reader = std::move(got_reader); + } + + void Roundtrip( + std::shared_ptr reader, + std::function&)> check_func) { + ArrowArrayStream c_stream; + + // NOTE: ReleaseCallback<> is not immediately usable with ArrowArrayStream, + // because get_next and get_schema need the original private_data. + std::weak_ptr weak_reader(reader); + ASSERT_EQ(weak_reader.use_count(), 1); // Expiration check will fail otherwise + + ASSERT_OK(ExportRecordBatchReader(std::move(reader), &c_stream)); + ASSERT_FALSE(ArrowArrayStreamIsReleased(&c_stream)); + + { + ASSERT_OK_AND_ASSIGN(auto new_reader, ImportRecordBatchReader(&c_stream)); + // Stream was moved + ASSERT_TRUE(ArrowArrayStreamIsReleased(&c_stream)); + ASSERT_FALSE(weak_reader.expired()); + + check_func(new_reader); + } + // Stream was released when `new_reader` was destroyed + ASSERT_TRUE(weak_reader.expired()); + } + + void AssertReaderNext(const std::shared_ptr& reader, + const RecordBatch& expected) { + ASSERT_OK_AND_ASSIGN(auto batch, reader->Next()); + ASSERT_NE(batch, nullptr); + AssertBatchesEqual(expected, *batch); + } + + void AssertReaderEnd(const std::shared_ptr& reader) { + ASSERT_OK_AND_ASSIGN(auto batch, reader->Next()); + ASSERT_EQ(batch, nullptr); + } +}; + +TEST_F(TestArrayStreamRoundtrip, Simple) { + auto orig_schema = arrow::schema({field("ints", int32())}); + auto batches = MakeBatches(orig_schema, {ArrayFromJSON(int32(), "[1, 2]"), + ArrayFromJSON(int32(), "[4, 5, null]")}); + + ASSERT_OK_AND_ASSIGN(auto reader, RecordBatchReader::Make(batches, orig_schema)); + + Roundtrip(std::move(reader), [&](const std::shared_ptr& reader) { + AssertSchemaEqual(*orig_schema, *reader->schema()); + AssertReaderNext(reader, *batches[0]); + AssertReaderNext(reader, *batches[1]); + AssertReaderEnd(reader); + AssertReaderEnd(reader); + }); +} + +TEST_F(TestArrayStreamRoundtrip, Errors) { + auto reader = std::make_shared( + Status::Invalid("roundtrip error example")); + + Roundtrip(std::move(reader), [&](const std::shared_ptr& reader) { + auto status = reader->Next().status(); + ASSERT_RAISES(Invalid, status); + ASSERT_THAT(status.message(), ::testing::HasSubstr("roundtrip error example")); + }); +} + } // namespace arrow diff --git a/cpp/src/arrow/c/helpers.h b/cpp/src/arrow/c/helpers.h index a1a1240dd756..a5c1f6fe4bab 100644 --- a/cpp/src/arrow/c/helpers.h +++ b/cpp/src/arrow/c/helpers.h @@ -82,6 +82,36 @@ inline void ArrowArrayRelease(struct ArrowArray* array) { } } +/// Query whether the C array stream is released +inline int ArrowArrayStreamIsReleased(const struct ArrowArrayStream* stream) { + return stream->release == NULL; +} + +/// Mark the C array stream released (for use in release callbacks) +inline void ArrowArrayStreamMarkReleased(struct ArrowArrayStream* stream) { + stream->release = NULL; +} + +/// Move the C array stream from `src` to `dest` +/// +/// Note `dest` must *not* point to a valid stream already, otherwise there +/// will be a memory leak. +inline void ArrowArrayStreamMove(struct ArrowArrayStream* src, + struct ArrowArrayStream* dest) { + assert(dest != src); + assert(!ArrowArrayStreamIsReleased(src)); + memcpy(dest, src, sizeof(struct ArrowArrayStream)); + ArrowArrayStreamMarkReleased(src); +} + +/// Release the C array stream, if necessary, by calling its release callback +inline void ArrowArrayStreamRelease(struct ArrowArrayStream* stream) { + if (!ArrowArrayStreamIsReleased(stream)) { + stream->release(stream); + assert(ArrowArrayStreamIsReleased(stream)); + } +} + #ifdef __cplusplus } #endif diff --git a/cpp/src/arrow/c/util_internal.h b/cpp/src/arrow/c/util_internal.h index 3ece5245205a..6a33be9b0da8 100644 --- a/cpp/src/arrow/c/util_internal.h +++ b/cpp/src/arrow/c/util_internal.h @@ -34,6 +34,12 @@ struct ArrayExportTraits { static constexpr auto ReleaseFunc = &ArrowArrayRelease; }; +struct ArrayStreamExportTraits { + typedef struct ArrowArrayStream CType; + static constexpr auto IsReleasedFunc = &ArrowArrayStreamIsReleased; + static constexpr auto ReleaseFunc = &ArrowArrayStreamRelease; +}; + // A RAII-style object to release a C Array / Schema struct at block scope exit. template class ExportGuard { @@ -73,6 +79,7 @@ class ExportGuard { using SchemaExportGuard = ExportGuard; using ArrayExportGuard = ExportGuard; +using ArrayStreamExportGuard = ExportGuard; } // namespace internal } // namespace arrow diff --git a/cpp/src/arrow/compare.cc b/cpp/src/arrow/compare.cc index a4cea6f4da40..421ec1392422 100644 --- a/cpp/src/arrow/compare.cc +++ b/cpp/src/arrow/compare.cc @@ -110,11 +110,12 @@ inline bool FloatingApproxEquals(const NumericArray& left, if (opts.nans_equal()) { return BaseFloatingEquals(left, right, [epsilon](T x, T y) -> bool { - return (fabs(x - y) <= epsilon) || (std::isnan(x) && std::isnan(y)); + return (fabs(x - y) <= epsilon) || (x == y) || (std::isnan(x) && std::isnan(y)); }); } else { - return BaseFloatingEquals( - left, right, [epsilon](T x, T y) -> bool { return fabs(x - y) <= epsilon; }); + return BaseFloatingEquals(left, right, [epsilon](T x, T y) -> bool { + return (fabs(x - y) <= epsilon) || (x == y); + }); } } @@ -861,7 +862,9 @@ class TypeEqualsVisitor { class ScalarEqualsVisitor { public: - explicit ScalarEqualsVisitor(const Scalar& right) : right_(right), result_(false) {} + explicit ScalarEqualsVisitor(const Scalar& right, + const EqualOptions& opts = EqualOptions::Defaults()) + : right_(right), result_(false), options_(opts) {} Status Visit(const NullScalar& left) { result_ = true; @@ -874,9 +877,26 @@ class ScalarEqualsVisitor { return Status::OK(); } + template + typename std::enable_if::value || + std::is_base_of::value, + Status>::type + Visit(const T& left_) { + const auto& right = checked_cast(right_); + if (options_.nans_equal()) { + result_ = right.value == left_.value || + (std::isnan(right.value) && std::isnan(left_.value)); + } else { + result_ = right.value == left_.value; + } + return Status::OK(); + } + template typename std::enable_if< - std::is_base_of, T>::value || + (std::is_base_of, T>::value && + !std::is_base_of::value && + !std::is_base_of::value) || std::is_base_of, T>::value, Status>::type Visit(const T& left_) { @@ -967,6 +987,7 @@ class ScalarEqualsVisitor { protected: const Scalar& right_; bool result_; + const EqualOptions options_; }; Status PrintDiff(const Array& left, const Array& right, std::ostream* os) { @@ -1385,7 +1406,7 @@ bool TypeEquals(const DataType& left, const DataType& right, bool check_metadata } } -bool ScalarEquals(const Scalar& left, const Scalar& right) { +bool ScalarEquals(const Scalar& left, const Scalar& right, const EqualOptions& options) { bool are_equal = false; if (&left == &right) { are_equal = true; @@ -1394,7 +1415,7 @@ bool ScalarEquals(const Scalar& left, const Scalar& right) { } else if (left.is_valid != right.is_valid) { are_equal = false; } else { - ScalarEqualsVisitor visitor(right); + ScalarEqualsVisitor visitor(right, options); auto error = VisitScalarInline(left, &visitor); DCHECK_OK(error); are_equal = visitor.result(); diff --git a/cpp/src/arrow/compare.h b/cpp/src/arrow/compare.h index abcf39a62e57..f7899b7c5c67 100644 --- a/cpp/src/arrow/compare.h +++ b/cpp/src/arrow/compare.h @@ -111,6 +111,8 @@ bool ARROW_EXPORT TypeEquals(const DataType& left, const DataType& right, /// Returns true if scalars are equal /// \param[in] left a Scalar /// \param[in] right a Scalar -bool ARROW_EXPORT ScalarEquals(const Scalar& left, const Scalar& right); +/// \param[in] options comparison options +bool ARROW_EXPORT ScalarEquals(const Scalar& left, const Scalar& right, + const EqualOptions& options = EqualOptions::Defaults()); } // namespace arrow diff --git a/cpp/src/arrow/compute/api_aggregate.cc b/cpp/src/arrow/compute/api_aggregate.cc index 95e859e20eea..53ee5b9a2b23 100644 --- a/cpp/src/arrow/compute/api_aggregate.cc +++ b/cpp/src/arrow/compute/api_aggregate.cc @@ -41,5 +41,19 @@ Result MinMax(const Datum& value, const MinMaxOptions& options, ExecConte return CallFunction("min_max", {value}, &options, ctx); } +Result Mode(const Datum& value, ExecContext* ctx) { + return CallFunction("mode", {value}, ctx); +} + +Result Stddev(const Datum& value, const VarianceOptions& options, + ExecContext* ctx) { + return CallFunction("stddev", {value}, &options, ctx); +} + +Result Variance(const Datum& value, const VarianceOptions& options, + ExecContext* ctx) { + return CallFunction("variance", {value}, &options, ctx); +} + } // namespace compute } // namespace arrow diff --git a/cpp/src/arrow/compute/api_aggregate.h b/cpp/src/arrow/compute/api_aggregate.h index 72b31082c103..41c6e1d613a5 100644 --- a/cpp/src/arrow/compute/api_aggregate.h +++ b/cpp/src/arrow/compute/api_aggregate.h @@ -66,7 +66,7 @@ struct ARROW_EXPORT MinMaxOptions : public FunctionOptions { /// Skip null values SKIP = 0, /// Any nulls will result in null output - OUTPUT_NULL + EMIT_NULL }; explicit MinMaxOptions(enum Mode null_handling = SKIP) : null_handling(null_handling) {} @@ -76,6 +76,18 @@ struct ARROW_EXPORT MinMaxOptions : public FunctionOptions { enum Mode null_handling = SKIP; }; +/// \brief Control Delta Degrees of Freedom (ddof) of Variance and Stddev kernel +/// +/// The divisor used in calculations is N - ddof, where N is the number of elements. +/// By default, ddof is zero, and population variance or stddev is returned. +struct ARROW_EXPORT VarianceOptions : public FunctionOptions { + explicit VarianceOptions(int ddof = 0) : ddof(ddof) {} + + static VarianceOptions Defaults() { return VarianceOptions{}; } + + int ddof = 0; +}; + /// @} /// \brief Count non-null (or null) values in an array. @@ -116,7 +128,7 @@ Result Sum(const Datum& value, ExecContext* ctx = NULLPTR); /// \brief Calculate the min / max of a numeric array /// /// This function returns both the min and max as a struct scalar, with type -/// struct, where T is ht einput type +/// struct, where T is the input type /// /// \param[in] value input datum, expecting Array or ChunkedArray /// \param[in] options see MinMaxOptions for more information @@ -130,22 +142,48 @@ Result MinMax(const Datum& value, const MinMaxOptions& options = MinMaxOptions::Defaults(), ExecContext* ctx = NULLPTR); -/// \brief Calculate the min / max of a numeric array. +/// \brief Calculate the modal (most common) value of a numeric array /// -/// This function returns both the min and max as a collection. The resulting -/// datum thus consists of two scalar datums: {Datum(min), Datum(max)} +/// This function returns both mode and count as a struct scalar, with type +/// struct, where T is the input type. +/// If there is more than one such value, the smallest one is returned. /// -/// \param[in] array input array -/// \param[in] options see MinMaxOptions for more information +/// \param[in] value input datum, expecting Array or ChunkedArray /// \param[in] ctx the function execution context, optional -/// \return resulting datum containing a {min, max} collection +/// \return resulting datum as a struct scalar /// -/// \since 1.0.0 +/// \since 2.0.0 /// \note API not yet finalized ARROW_EXPORT -Result MinMax(const Array& array, - const MinMaxOptions& options = MinMaxOptions::Defaults(), +Result Mode(const Datum& value, ExecContext* ctx = NULLPTR); + +/// \brief Calculate the standard deviation of a numeric array +/// +/// \param[in] value input datum, expecting Array or ChunkedArray +/// \param[in] options see VarianceOptions for more information +/// \param[in] ctx the function execution context, optional +/// \return datum of the computed standard deviation as a DoubleScalar +/// +/// \since 2.0.0 +/// \note API not yet finalized +ARROW_EXPORT +Result Stddev(const Datum& value, + const VarianceOptions& options = VarianceOptions::Defaults(), ExecContext* ctx = NULLPTR); +/// \brief Calculate the variance of a numeric array +/// +/// \param[in] value input datum, expecting Array or ChunkedArray +/// \param[in] options see VarianceOptions for more information +/// \param[in] ctx the function execution context, optional +/// \return datum of the computed variance as a DoubleScalar +/// +/// \since 2.0.0 +/// \note API not yet finalized +ARROW_EXPORT +Result Variance(const Datum& value, + const VarianceOptions& options = VarianceOptions::Defaults(), + ExecContext* ctx = NULLPTR); + } // namespace compute } // namespace arrow diff --git a/cpp/src/arrow/compute/api_scalar.cc b/cpp/src/arrow/compute/api_scalar.cc index 9a9110309993..353151eade2c 100644 --- a/cpp/src/arrow/compute/api_scalar.cc +++ b/cpp/src/arrow/compute/api_scalar.cc @@ -51,6 +51,7 @@ namespace compute { SCALAR_ARITHMETIC_BINARY(Add, "add", "add_checked") SCALAR_ARITHMETIC_BINARY(Subtract, "subtract", "subtract_checked") SCALAR_ARITHMETIC_BINARY(Multiply, "multiply", "multiply_checked") +SCALAR_ARITHMETIC_BINARY(Divide, "divide", "divide_checked") // ---------------------------------------------------------------------- // Set-related operations diff --git a/cpp/src/arrow/compute/api_scalar.h b/cpp/src/arrow/compute/api_scalar.h index 80e3ebb98b3c..fc9dc35490e6 100644 --- a/cpp/src/arrow/compute/api_scalar.h +++ b/cpp/src/arrow/compute/api_scalar.h @@ -129,6 +129,20 @@ Result Multiply(const Datum& left, const Datum& right, ArithmeticOptions options = ArithmeticOptions(), ExecContext* ctx = NULLPTR); +/// \brief Divide two values. Array values must be the same length. If either +/// argument is null the result will be null. For integer types, if there is +/// a zero divisor, an error will be raised. +/// +/// \param[in] left the dividend +/// \param[in] right the divisor +/// \param[in] options arithmetic options (enable/disable overflow checking), optional +/// \param[in] ctx the function execution context, optional +/// \return the elementwise quotient +ARROW_EXPORT +Result Divide(const Datum& left, const Datum& right, + ArithmeticOptions options = ArithmeticOptions(), + ExecContext* ctx = NULLPTR); + /// \brief Compare a numeric array with a scalar. /// /// \param[in] left datum to compare, must be an Array diff --git a/cpp/src/arrow/compute/api_vector.cc b/cpp/src/arrow/compute/api_vector.cc index 9a36714d107f..1f6972860aea 100644 --- a/cpp/src/arrow/compute/api_vector.cc +++ b/cpp/src/arrow/compute/api_vector.cc @@ -21,13 +21,18 @@ #include #include +#include "arrow/array/array_nested.h" #include "arrow/array/builder_primitive.h" #include "arrow/compute/exec.h" #include "arrow/datum.h" #include "arrow/record_batch.h" #include "arrow/result.h" +#include "arrow/util/checked_cast.h" namespace arrow { + +using internal::checked_pointer_cast; + namespace compute { // ---------------------------------------------------------------------- @@ -60,9 +65,9 @@ const char kCountsFieldName[] = "counts"; const int32_t kValuesFieldIndex = 0; const int32_t kCountsFieldIndex = 1; -Result> ValueCounts(const Datum& value, ExecContext* ctx) { +Result> ValueCounts(const Datum& value, ExecContext* ctx) { ARROW_ASSIGN_OR_RAISE(Datum result, CallFunction("value_counts", {value}, ctx)); - return result.make_array(); + return checked_pointer_cast(result.make_array()); } // ---------------------------------------------------------------------- diff --git a/cpp/src/arrow/compute/api_vector.h b/cpp/src/arrow/compute/api_vector.h index 3aa3434c0982..2c77e8ee1550 100644 --- a/cpp/src/arrow/compute/api_vector.h +++ b/cpp/src/arrow/compute/api_vector.h @@ -59,7 +59,7 @@ struct ARROW_EXPORT TakeOptions : public FunctionOptions { }; /// \brief Partitioning options for NthToIndices -struct PartitionNthOptions : public FunctionOptions { +struct ARROW_EXPORT PartitionNthOptions : public FunctionOptions { explicit PartitionNthOptions(int64_t pivot) : pivot(pivot) {} /// The index into the equivalent sorted array of the partition pivot element. @@ -202,8 +202,8 @@ ARROW_EXPORT extern const int32_t kCountsFieldIndex; /// \since 1.0.0 /// \note API not yet finalized ARROW_EXPORT -Result> ValueCounts(const Datum& value, - ExecContext* ctx = NULLPTR); +Result> ValueCounts(const Datum& value, + ExecContext* ctx = NULLPTR); /// \brief Dictionary-encode values in an array-like object /// \param[in] data array-like input diff --git a/cpp/src/arrow/compute/exec.cc b/cpp/src/arrow/compute/exec.cc index fbc3693e5be8..71bbc34187e6 100644 --- a/cpp/src/arrow/compute/exec.cc +++ b/cpp/src/arrow/compute/exec.cc @@ -885,7 +885,7 @@ class ScalarAggExecutor : public FunctionExecutorImpl { kernel_->consume(&batch_ctx, batch); ARROW_CTX_RETURN_IF_ERROR(&batch_ctx); - kernel_->merge(&kernel_ctx_, *batch_state, state_.get()); + kernel_->merge(&kernel_ctx_, std::move(*batch_state), state_.get()); ARROW_CTX_RETURN_IF_ERROR(&kernel_ctx_); return Status::OK(); } @@ -952,9 +952,6 @@ Result CallFunction(const std::string& func_name, const std::vector func, ctx->func_registry()->GetFunction(func_name)); - if (options == nullptr) { - options = func->default_options(); - } return func->Execute(args, options, ctx); } diff --git a/cpp/src/arrow/compute/function.cc b/cpp/src/arrow/compute/function.cc index 41c3e360a07f..91da1ae76ffa 100644 --- a/cpp/src/arrow/compute/function.cc +++ b/cpp/src/arrow/compute/function.cc @@ -103,6 +103,9 @@ Result DispatchExactImpl(const Function& func, Result Function::Execute(const std::vector& args, const FunctionOptions* options, ExecContext* ctx) const { + if (options == nullptr) { + options = default_options(); + } if (ctx == nullptr) { ExecContext default_ctx; return Execute(args, options, &default_ctx); @@ -189,6 +192,9 @@ Result MetaFunction::Execute(const std::vector& args, const FunctionOptions* options, ExecContext* ctx) const { RETURN_NOT_OK(CheckArity(static_cast(args.size()))); + if (options == nullptr) { + options = default_options(); + } return ExecuteImpl(args, options, ctx); } diff --git a/cpp/src/arrow/compute/function.h b/cpp/src/arrow/compute/function.h index 93a200ee2122..0b800ade6c9b 100644 --- a/cpp/src/arrow/compute/function.h +++ b/cpp/src/arrow/compute/function.h @@ -65,7 +65,8 @@ struct ARROW_EXPORT Arity { /// invoking the function static Arity VarArgs(int min_args = 0) { return Arity(min_args, true); } - explicit Arity(int num_args, bool is_varargs = false) + // NOTE: the 0-argument form (default constructor) is required for Cython + explicit Arity(int num_args = 0, bool is_varargs = false) : num_args(num_args), is_varargs(is_varargs) {} /// The number of required arguments (or the minimum number for varargs @@ -124,8 +125,7 @@ class ARROW_EXPORT Function { /// kernel dispatch, batch iteration, and memory allocation details taken /// care of. /// - /// Function implementations may assume that options is non-null and valid - /// or to forgo options and accept only nullptr for that argument. + /// If the `options` pointer is null, then `default_options()` will be used. /// /// This function can be overridden in subclasses. virtual Result Execute(const std::vector& args, diff --git a/cpp/src/arrow/compute/kernel.cc b/cpp/src/arrow/compute/kernel.cc index 1788eb72963f..88b42716fa2c 100644 --- a/cpp/src/arrow/compute/kernel.cc +++ b/cpp/src/arrow/compute/kernel.cc @@ -281,6 +281,9 @@ std::string InputType::ToString() const { } ss << "["; switch (kind_) { + case InputType::ANY_TYPE: + ss << "any"; + break; case InputType::EXACT_TYPE: ss << type_->ToString(); break; @@ -303,6 +306,8 @@ bool InputType::Equals(const InputType& other) const { return false; } switch (kind_) { + case InputType::ANY_TYPE: + return true; case InputType::EXACT_TYPE: return type_->Equals(*other.type_); case InputType::USE_TYPE_MATCHER: diff --git a/cpp/src/arrow/compute/kernel.h b/cpp/src/arrow/compute/kernel.h index 3fb6947107e6..67cb5df79081 100644 --- a/cpp/src/arrow/compute/kernel.h +++ b/cpp/src/arrow/compute/kernel.h @@ -664,7 +664,7 @@ struct VectorKernel : public ArrayKernel { using ScalarAggregateConsume = std::function; using ScalarAggregateMerge = - std::function; + std::function; // Finalize returns Datum to permit multiple return values using ScalarAggregateFinalize = std::function; diff --git a/cpp/src/arrow/compute/kernel_test.cc b/cpp/src/arrow/compute/kernel_test.cc index 2eb7fd114491..df18fceaa208 100644 --- a/cpp/src/arrow/compute/kernel_test.cc +++ b/cpp/src/arrow/compute/kernel_test.cc @@ -141,6 +141,15 @@ TEST(InputType, Constructors) { InputType ty7(match::TimestampTypeUnit(TimeUnit::MICRO)); ASSERT_EQ("any[timestamp(us)]", ty7.ToString()); + + InputType ty8; + InputType ty9(ValueDescr::ANY); + InputType ty10(ValueDescr::ARRAY); + InputType ty11(ValueDescr::SCALAR); + ASSERT_EQ("any[any]", ty8.ToString()); + ASSERT_EQ("any[any]", ty9.ToString()); + ASSERT_EQ("array[any]", ty10.ToString()); + ASSERT_EQ("scalar[any]", ty11.ToString()); } TEST(InputType, Equals) { diff --git a/cpp/src/arrow/compute/kernels/aggregate_basic.cc b/cpp/src/arrow/compute/kernels/aggregate_basic.cc index 2f3cdda3daa4..562e17485fde 100644 --- a/cpp/src/arrow/compute/kernels/aggregate_basic.cc +++ b/cpp/src/arrow/compute/kernels/aggregate_basic.cc @@ -15,12 +15,11 @@ // specific language governing permissions and limitations // under the License. -#include - #include "arrow/compute/api_aggregate.h" #include "arrow/compute/kernels/aggregate_basic_internal.h" #include "arrow/compute/kernels/aggregate_internal.h" #include "arrow/compute/kernels/common.h" +#include "arrow/util/cpu_info.h" #include "arrow/util/make_unique.h" namespace arrow { @@ -31,8 +30,8 @@ void AggregateConsume(KernelContext* ctx, const ExecBatch& batch) { checked_cast(ctx->state())->Consume(ctx, batch); } -void AggregateMerge(KernelContext* ctx, const KernelState& src, KernelState* dst) { - checked_cast(dst)->MergeFrom(ctx, src); +void AggregateMerge(KernelContext* ctx, KernelState&& src, KernelState* dst) { + checked_cast(dst)->MergeFrom(ctx, std::move(src)); } void AggregateFinalize(KernelContext* ctx, Datum* out) { @@ -52,7 +51,7 @@ struct CountImpl : public ScalarAggregator { this->non_nulls += input.length - nulls; } - void MergeFrom(KernelContext*, const KernelState& src) override { + void MergeFrom(KernelContext*, KernelState&& src) override { const auto& other_state = checked_cast(src); this->non_nulls += other_state.non_nulls; this->nulls += other_state.nulls; @@ -131,218 +130,15 @@ std::unique_ptr MeanInit(KernelContext* ctx, const KernelInitArgs& // ---------------------------------------------------------------------- // MinMax implementation -template -struct MinMaxState {}; - -template -struct MinMaxState> { - using ThisType = MinMaxState; - using T = typename ArrowType::c_type; - - ThisType& operator+=(const ThisType& rhs) { - this->has_nulls |= rhs.has_nulls; - this->has_values |= rhs.has_values; - this->min = this->min && rhs.min; - this->max = this->max || rhs.max; - return *this; - } - - void MergeOne(T value) { - this->min = this->min && value; - this->max = this->max || value; - } - - T min = true; - T max = false; - bool has_nulls = false; - bool has_values = false; -}; - -template -struct MinMaxState> { - using ThisType = MinMaxState; - using T = typename ArrowType::c_type; - - ThisType& operator+=(const ThisType& rhs) { - this->has_nulls |= rhs.has_nulls; - this->has_values |= rhs.has_values; - this->min = std::min(this->min, rhs.min); - this->max = std::max(this->max, rhs.max); - return *this; - } - - void MergeOne(T value) { - this->min = std::min(this->min, value); - this->max = std::max(this->max, value); - } - - T min = std::numeric_limits::max(); - T max = std::numeric_limits::min(); - bool has_nulls = false; - bool has_values = false; -}; - -template -struct MinMaxState> { - using ThisType = MinMaxState; - using T = typename ArrowType::c_type; - - ThisType& operator+=(const ThisType& rhs) { - this->has_nulls |= rhs.has_nulls; - this->has_values |= rhs.has_values; - this->min = std::fmin(this->min, rhs.min); - this->max = std::fmax(this->max, rhs.max); - return *this; - } - - void MergeOne(T value) { - this->min = std::fmin(this->min, value); - this->max = std::fmax(this->max, value); - } - - T min = std::numeric_limits::infinity(); - T max = -std::numeric_limits::infinity(); - bool has_nulls = false; - bool has_values = false; -}; - -template -struct MinMaxImpl : public ScalarAggregator { - using ArrayType = typename TypeTraits::ArrayType; - using ThisType = MinMaxImpl; - using StateType = MinMaxState; - - MinMaxImpl(const std::shared_ptr& out_type, const MinMaxOptions& options) - : out_type(out_type), options(options) {} - - void Consume(KernelContext*, const ExecBatch& batch) override { - StateType local; - - ArrayType arr(batch[0].array()); - - const auto null_count = arr.null_count(); - local.has_nulls = null_count > 0; - local.has_values = (arr.length() - null_count) > 0; - - if (local.has_nulls && options.null_handling == MinMaxOptions::OUTPUT_NULL) { - this->state = local; - return; - } - - if (local.has_nulls) { - BitmapReader reader(arr.null_bitmap_data(), arr.offset(), arr.length()); - for (int64_t i = 0; i < arr.length(); i++) { - if (reader.IsSet()) { - local.MergeOne(arr.Value(i)); - } - reader.Next(); - } - } else { - for (int64_t i = 0; i < arr.length(); i++) { - local.MergeOne(arr.Value(i)); - } - } - this->state = local; - } - - void MergeFrom(KernelContext*, const KernelState& src) override { - const auto& other = checked_cast(src); - this->state += other.state; - } - - void Finalize(KernelContext*, Datum* out) override { - using ScalarType = typename TypeTraits::ScalarType; - - std::vector> values; - if (!state.has_values || - (state.has_nulls && options.null_handling == MinMaxOptions::OUTPUT_NULL)) { - // (null, null) - values = {std::make_shared(), std::make_shared()}; - } else { - values = {std::make_shared(state.min), - std::make_shared(state.max)}; - } - out->value = std::make_shared(std::move(values), this->out_type); - } - - std::shared_ptr out_type; - MinMaxOptions options; - MinMaxState state; -}; - -struct BooleanMinMaxImpl : public MinMaxImpl { - using MinMaxImpl::MinMaxImpl; - - void Consume(KernelContext*, const ExecBatch& batch) override { - StateType local; - ArrayType arr(batch[0].array()); - - const auto arr_length = arr.length(); - const auto null_count = arr.null_count(); - const auto valid_count = arr_length - null_count; - - local.has_nulls = null_count > 0; - local.has_values = valid_count > 0; - if (local.has_nulls && options.null_handling == MinMaxOptions::OUTPUT_NULL) { - this->state = local; - return; - } - - const auto true_count = arr.true_count(); - const auto false_count = valid_count - true_count; - local.max = true_count > 0; - local.min = false_count == 0; - - this->state = local; - } -}; - -struct MinMaxInitState { - std::unique_ptr state; - KernelContext* ctx; - const DataType& in_type; - const std::shared_ptr& out_type; - const MinMaxOptions& options; - - MinMaxInitState(KernelContext* ctx, const DataType& in_type, - const std::shared_ptr& out_type, const MinMaxOptions& options) - : ctx(ctx), in_type(in_type), out_type(out_type), options(options) {} - - Status Visit(const DataType&) { - return Status::NotImplemented("No min/max implemented"); - } - - Status Visit(const HalfFloatType&) { - return Status::NotImplemented("No sum implemented"); - } - - Status Visit(const BooleanType&) { - state.reset(new BooleanMinMaxImpl(out_type, options)); - return Status::OK(); - } - - template - enable_if_number Visit(const Type&) { - state.reset(new MinMaxImpl(out_type, options)); - return Status::OK(); - } - - std::unique_ptr Create() { - ctx->SetStatus(VisitTypeInline(in_type, this)); - return std::move(state); - } -}; - std::unique_ptr MinMaxInit(KernelContext* ctx, const KernelInitArgs& args) { - MinMaxInitState visitor(ctx, *args.inputs[0].type, - args.kernel->signature->out_type().type(), - static_cast(*args.options)); + MinMaxInitState visitor( + ctx, *args.inputs[0].type, args.kernel->signature->out_type().type(), + static_cast(*args.options)); return visitor.Create(); } void AddAggKernel(std::shared_ptr sig, KernelInit init, - ScalarAggregateFunction* func, - SimdLevel::type simd_level = SimdLevel::NONE) { + ScalarAggregateFunction* func, SimdLevel::type simd_level) { ScalarAggregateKernel kernel(std::move(sig), init, AggregateConsume, AggregateMerge, AggregateFinalize); // Set the simd level @@ -363,8 +159,7 @@ void AddBasicAggKernels(KernelInit init, void AddMinMaxKernels(KernelInit init, const std::vector>& types, - ScalarAggregateFunction* func, - SimdLevel::type simd_level = SimdLevel::NONE) { + ScalarAggregateFunction* func, SimdLevel::type simd_level) { for (const auto& ty : types) { // array[T] -> scalar[struct] auto out_ty = struct_({field("min", ty), field("max", ty)}); @@ -396,11 +191,16 @@ void RegisterScalarAggregateBasic(FunctionRegistry* registry) { aggregate::AddBasicAggKernels(aggregate::SumInit, FloatingPointTypes(), float64(), func.get()); // Add the SIMD variants for sum + auto cpu_info = arrow::internal::CpuInfo::GetInstance(); #if defined(ARROW_HAVE_RUNTIME_AVX2) - aggregate::AddSumAvx2AggKernels(func.get()); + if (cpu_info->IsSupported(arrow::internal::CpuInfo::AVX2)) { + aggregate::AddSumAvx2AggKernels(func.get()); + } #endif #if defined(ARROW_HAVE_RUNTIME_AVX512) - aggregate::AddSumAvx512AggKernels(func.get()); + if (cpu_info->IsSupported(arrow::internal::CpuInfo::AVX512)) { + aggregate::AddSumAvx512AggKernels(func.get()); + } #endif DCHECK_OK(registry->AddFunction(std::move(func))); @@ -410,10 +210,14 @@ void RegisterScalarAggregateBasic(FunctionRegistry* registry) { func.get()); // Add the SIMD variants for mean #if defined(ARROW_HAVE_RUNTIME_AVX2) - aggregate::AddMeanAvx2AggKernels(func.get()); + if (cpu_info->IsSupported(arrow::internal::CpuInfo::AVX2)) { + aggregate::AddMeanAvx2AggKernels(func.get()); + } #endif #if defined(ARROW_HAVE_RUNTIME_AVX512) - aggregate::AddMeanAvx512AggKernels(func.get()); + if (cpu_info->IsSupported(arrow::internal::CpuInfo::AVX512)) { + aggregate::AddMeanAvx512AggKernels(func.get()); + } #endif DCHECK_OK(registry->AddFunction(std::move(func))); @@ -422,7 +226,23 @@ void RegisterScalarAggregateBasic(FunctionRegistry* registry) { &default_minmax_options); aggregate::AddMinMaxKernels(aggregate::MinMaxInit, {boolean()}, func.get()); aggregate::AddMinMaxKernels(aggregate::MinMaxInit, NumericTypes(), func.get()); + // Add the SIMD variants for min max +#if defined(ARROW_HAVE_RUNTIME_AVX2) + if (cpu_info->IsSupported(arrow::internal::CpuInfo::AVX2)) { + aggregate::AddMinMaxAvx2AggKernels(func.get()); + } +#endif +#if defined(ARROW_HAVE_RUNTIME_AVX512) + if (cpu_info->IsSupported(arrow::internal::CpuInfo::AVX512)) { + aggregate::AddMinMaxAvx512AggKernels(func.get()); + } +#endif + DCHECK_OK(registry->AddFunction(std::move(func))); + + DCHECK_OK(registry->AddFunction(aggregate::AddModeAggKernels())); + DCHECK_OK(registry->AddFunction(aggregate::AddStddevAggKernels())); + DCHECK_OK(registry->AddFunction(aggregate::AddVarianceAggKernels())); } } // namespace internal diff --git a/cpp/src/arrow/compute/kernels/aggregate_sum_avx2.cc b/cpp/src/arrow/compute/kernels/aggregate_basic_avx2.cc similarity index 80% rename from cpp/src/arrow/compute/kernels/aggregate_sum_avx2.cc rename to cpp/src/arrow/compute/kernels/aggregate_basic_avx2.cc index 2811c4cd8653..e0c1118c7149 100644 --- a/cpp/src/arrow/compute/kernels/aggregate_sum_avx2.cc +++ b/cpp/src/arrow/compute/kernels/aggregate_basic_avx2.cc @@ -67,6 +67,17 @@ std::unique_ptr MeanInitAvx2(KernelContext* ctx, return visitor.Create(); } +// ---------------------------------------------------------------------- +// MinMax implementation + +std::unique_ptr MinMaxInitAvx2(KernelContext* ctx, + const KernelInitArgs& args) { + MinMaxInitState visitor( + ctx, *args.inputs[0].type, args.kernel->signature->out_type().type(), + static_cast(*args.options)); + return visitor.Create(); +} + void AddSumAvx2AggKernels(ScalarAggregateFunction* func) { AddBasicAggKernels(SumInitAvx2, internal::SignedIntTypes(), int64(), func, SimdLevel::AVX2); @@ -81,6 +92,12 @@ void AddMeanAvx2AggKernels(ScalarAggregateFunction* func) { SimdLevel::AVX2); } +void AddMinMaxAvx2AggKernels(ScalarAggregateFunction* func) { + // Enable int types for AVX2 variants. + // No auto vectorize for float/double as it use fmin/fmax which has NaN handling. + AddMinMaxKernels(MinMaxInitAvx2, internal::IntTypes(), func, SimdLevel::AVX2); +} + } // namespace aggregate } // namespace compute } // namespace arrow diff --git a/cpp/src/arrow/compute/kernels/aggregate_sum_avx512.cc b/cpp/src/arrow/compute/kernels/aggregate_basic_avx512.cc similarity index 80% rename from cpp/src/arrow/compute/kernels/aggregate_sum_avx512.cc rename to cpp/src/arrow/compute/kernels/aggregate_basic_avx512.cc index 00408027e1ff..c2c748d3af7d 100644 --- a/cpp/src/arrow/compute/kernels/aggregate_sum_avx512.cc +++ b/cpp/src/arrow/compute/kernels/aggregate_basic_avx512.cc @@ -68,6 +68,17 @@ std::unique_ptr MeanInitAvx512(KernelContext* ctx, return visitor.Create(); } +// ---------------------------------------------------------------------- +// MinMax implementation + +std::unique_ptr MinMaxInitAvx512(KernelContext* ctx, + const KernelInitArgs& args) { + MinMaxInitState visitor( + ctx, *args.inputs[0].type, args.kernel->signature->out_type().type(), + static_cast(*args.options)); + return visitor.Create(); +} + void AddSumAvx512AggKernels(ScalarAggregateFunction* func) { AddBasicAggKernels(SumInitAvx512, internal::SignedIntTypes(), int64(), func, SimdLevel::AVX512); @@ -82,6 +93,12 @@ void AddMeanAvx512AggKernels(ScalarAggregateFunction* func) { SimdLevel::AVX512); } +void AddMinMaxAvx512AggKernels(ScalarAggregateFunction* func) { + // Enable 32/64 int types for avx512 variants, no advantage on 8/16 int. + AddMinMaxKernels(MinMaxInitAvx512, {int32(), uint32(), int64(), uint64()}, func, + SimdLevel::AVX512); +} + } // namespace aggregate } // namespace compute } // namespace arrow diff --git a/cpp/src/arrow/compute/kernels/aggregate_basic_internal.h b/cpp/src/arrow/compute/kernels/aggregate_basic_internal.h index e3014675bc28..2b0631ee2f20 100644 --- a/cpp/src/arrow/compute/kernels/aggregate_basic_internal.h +++ b/cpp/src/arrow/compute/kernels/aggregate_basic_internal.h @@ -17,6 +17,9 @@ #pragma once +#include + +#include "arrow/compute/api_aggregate.h" #include "arrow/compute/kernels/aggregate_internal.h" #include "arrow/compute/kernels/common.h" #include "arrow/util/align_util.h" @@ -28,29 +31,44 @@ namespace aggregate { struct ScalarAggregator : public KernelState { virtual void Consume(KernelContext* ctx, const ExecBatch& batch) = 0; - virtual void MergeFrom(KernelContext* ctx, const KernelState& src) = 0; + virtual void MergeFrom(KernelContext* ctx, KernelState&& src) = 0; virtual void Finalize(KernelContext* ctx, Datum* out) = 0; }; +void AddAggKernel(std::shared_ptr sig, KernelInit init, + ScalarAggregateFunction* func, + SimdLevel::type simd_level = SimdLevel::NONE); + void AddBasicAggKernels(KernelInit init, const std::vector>& types, std::shared_ptr out_ty, ScalarAggregateFunction* func, SimdLevel::type simd_level = SimdLevel::NONE); +void AddMinMaxKernels(KernelInit init, + const std::vector>& types, + ScalarAggregateFunction* func, + SimdLevel::type simd_level = SimdLevel::NONE); + // SIMD variants for kernels void AddSumAvx2AggKernels(ScalarAggregateFunction* func); void AddMeanAvx2AggKernels(ScalarAggregateFunction* func); +void AddMinMaxAvx2AggKernels(ScalarAggregateFunction* func); void AddSumAvx512AggKernels(ScalarAggregateFunction* func); void AddMeanAvx512AggKernels(ScalarAggregateFunction* func); +void AddMinMaxAvx512AggKernels(ScalarAggregateFunction* func); + +std::shared_ptr AddModeAggKernels(); +std::shared_ptr AddStddevAggKernels(); +std::shared_ptr AddVarianceAggKernels(); // ---------------------------------------------------------------------- // Sum implementation -template +template struct SumState { using SumType = typename FindAccumulatorType::Type; - using ThisType = SumState; + using ThisType = SumState; using T = typename TypeTraits::CType; using ArrayType = typename TypeTraits::ArrayType; @@ -211,10 +229,10 @@ struct SumState { } }; -template -struct SumState { +template +struct SumState { using SumType = typename FindAccumulatorType::Type; - using ThisType = SumState; + using ThisType = SumState; ThisType& operator+=(const ThisType& rhs) { this->count += rhs.count; @@ -233,10 +251,10 @@ struct SumState { typename SumType::c_type sum = 0; }; -template +template struct SumImpl : public ScalarAggregator { using ArrayType = typename TypeTraits::ArrayType; - using ThisType = SumImpl; + using ThisType = SumImpl; using SumType = typename FindAccumulatorType::Type; using OutputType = typename TypeTraits::ScalarType; @@ -244,7 +262,7 @@ struct SumImpl : public ScalarAggregator { this->state.Consume(ArrayType(batch[0].array())); } - void MergeFrom(KernelContext*, const KernelState& src) override { + void MergeFrom(KernelContext*, KernelState&& src) override { const auto& other = checked_cast(src); this->state += other.state; } @@ -257,11 +275,11 @@ struct SumImpl : public ScalarAggregator { } } - SumState state; + SumState state; }; -template -struct MeanImpl : public SumImpl { +template +struct MeanImpl : public SumImpl { void Finalize(KernelContext*, Datum* out) override { const bool is_valid = this->state.count > 0; const double divisor = static_cast(is_valid ? this->state.count : 1UL); @@ -306,6 +324,268 @@ struct SumLikeInit { } }; +// ---------------------------------------------------------------------- +// MinMax implementation + +template +struct MinMaxState {}; + +template +struct MinMaxState> { + using ThisType = MinMaxState; + using T = typename ArrowType::c_type; + + ThisType& operator+=(const ThisType& rhs) { + this->has_nulls |= rhs.has_nulls; + this->has_values |= rhs.has_values; + this->min = this->min && rhs.min; + this->max = this->max || rhs.max; + return *this; + } + + void MergeOne(T value) { + this->min = this->min && value; + this->max = this->max || value; + } + + T min = true; + T max = false; + bool has_nulls = false; + bool has_values = false; +}; + +template +struct MinMaxState> { + using ThisType = MinMaxState; + using T = typename ArrowType::c_type; + + ThisType& operator+=(const ThisType& rhs) { + this->has_nulls |= rhs.has_nulls; + this->has_values |= rhs.has_values; + this->min = std::min(this->min, rhs.min); + this->max = std::max(this->max, rhs.max); + return *this; + } + + void MergeOne(T value) { + this->min = std::min(this->min, value); + this->max = std::max(this->max, value); + } + + T min = std::numeric_limits::max(); + T max = std::numeric_limits::min(); + bool has_nulls = false; + bool has_values = false; +}; + +template +struct MinMaxState> { + using ThisType = MinMaxState; + using T = typename ArrowType::c_type; + + ThisType& operator+=(const ThisType& rhs) { + this->has_nulls |= rhs.has_nulls; + this->has_values |= rhs.has_values; + this->min = std::fmin(this->min, rhs.min); + this->max = std::fmax(this->max, rhs.max); + return *this; + } + + void MergeOne(T value) { + this->min = std::fmin(this->min, value); + this->max = std::fmax(this->max, value); + } + + T min = std::numeric_limits::infinity(); + T max = -std::numeric_limits::infinity(); + bool has_nulls = false; + bool has_values = false; +}; + +template +struct MinMaxImpl : public ScalarAggregator { + using ArrayType = typename TypeTraits::ArrayType; + using ThisType = MinMaxImpl; + using StateType = MinMaxState; + + MinMaxImpl(const std::shared_ptr& out_type, const MinMaxOptions& options) + : out_type(out_type), options(options) {} + + void Consume(KernelContext*, const ExecBatch& batch) override { + StateType local; + + ArrayType arr(batch[0].array()); + + const auto null_count = arr.null_count(); + local.has_nulls = null_count > 0; + local.has_values = (arr.length() - null_count) > 0; + + if (local.has_nulls && options.null_handling == MinMaxOptions::EMIT_NULL) { + this->state = local; + return; + } + + if (local.has_nulls) { + local += ConsumeWithNulls(arr); + } else { // All true values + for (int64_t i = 0; i < arr.length(); i++) { + local.MergeOne(arr.Value(i)); + } + } + this->state = local; + } + + void MergeFrom(KernelContext*, KernelState&& src) override { + const auto& other = checked_cast(src); + this->state += other.state; + } + + void Finalize(KernelContext*, Datum* out) override { + using ScalarType = typename TypeTraits::ScalarType; + + std::vector> values; + if (!state.has_values || + (state.has_nulls && options.null_handling == MinMaxOptions::EMIT_NULL)) { + // (null, null) + values = {std::make_shared(), std::make_shared()}; + } else { + values = {std::make_shared(state.min), + std::make_shared(state.max)}; + } + out->value = std::make_shared(std::move(values), this->out_type); + } + + std::shared_ptr out_type; + MinMaxOptions options; + MinMaxState state; + + private: + StateType ConsumeWithNulls(const ArrayType& arr) const { + StateType local; + const int64_t length = arr.length(); + int64_t offset = arr.offset(); + const uint8_t* bitmap = arr.null_bitmap_data(); + int64_t idx = 0; + + const auto p = arrow::internal::BitmapWordAlign<1>(bitmap, offset, length); + // First handle the leading bits + const int64_t leading_bits = p.leading_bits; + while (idx < leading_bits) { + if (BitUtil::GetBit(bitmap, offset)) { + local.MergeOne(arr.Value(idx)); + } + idx++; + offset++; + } + + // The aligned parts scanned with BitBlockCounter + arrow::internal::BitBlockCounter data_counter(bitmap, offset, length - leading_bits); + auto current_block = data_counter.NextWord(); + while (idx < length) { + if (current_block.AllSet()) { // All true values + int run_length = 0; + // Scan forward until a block that has some false values (or the end) + while (current_block.length > 0 && current_block.AllSet()) { + run_length += current_block.length; + current_block = data_counter.NextWord(); + } + for (int64_t i = 0; i < run_length; i++) { + local.MergeOne(arr.Value(idx + i)); + } + idx += run_length; + offset += run_length; + // The current_block already computed, advance to next loop + continue; + } else if (!current_block.NoneSet()) { // Some values are null + BitmapReader reader(arr.null_bitmap_data(), offset, current_block.length); + for (int64_t i = 0; i < current_block.length; i++) { + if (reader.IsSet()) { + local.MergeOne(arr.Value(idx + i)); + } + reader.Next(); + } + + idx += current_block.length; + offset += current_block.length; + } else { // All null values + idx += current_block.length; + offset += current_block.length; + } + current_block = data_counter.NextWord(); + } + + return local; + } +}; + +template +struct BooleanMinMaxImpl : public MinMaxImpl { + using StateType = MinMaxState; + using ArrayType = typename TypeTraits::ArrayType; + using MinMaxImpl::MinMaxImpl; + using MinMaxImpl::options; + + void Consume(KernelContext*, const ExecBatch& batch) override { + StateType local; + ArrayType arr(batch[0].array()); + + const auto arr_length = arr.length(); + const auto null_count = arr.null_count(); + const auto valid_count = arr_length - null_count; + + local.has_nulls = null_count > 0; + local.has_values = valid_count > 0; + if (local.has_nulls && options.null_handling == MinMaxOptions::EMIT_NULL) { + this->state = local; + return; + } + + const auto true_count = arr.true_count(); + const auto false_count = valid_count - true_count; + local.max = true_count > 0; + local.min = false_count == 0; + + this->state = local; + } +}; + +template +struct MinMaxInitState { + std::unique_ptr state; + KernelContext* ctx; + const DataType& in_type; + const std::shared_ptr& out_type; + const MinMaxOptions& options; + + MinMaxInitState(KernelContext* ctx, const DataType& in_type, + const std::shared_ptr& out_type, const MinMaxOptions& options) + : ctx(ctx), in_type(in_type), out_type(out_type), options(options) {} + + Status Visit(const DataType&) { + return Status::NotImplemented("No min/max implemented"); + } + + Status Visit(const HalfFloatType&) { + return Status::NotImplemented("No min/max implemented"); + } + + Status Visit(const BooleanType&) { + state.reset(new BooleanMinMaxImpl(out_type, options)); + return Status::OK(); + } + + template + enable_if_number Visit(const Type&) { + state.reset(new MinMaxImpl(out_type, options)); + return Status::OK(); + } + + std::unique_ptr Create() { + ctx->SetStatus(VisitTypeInline(in_type, this)); + return std::move(state); + } +}; + } // namespace aggregate } // namespace compute } // namespace arrow diff --git a/cpp/src/arrow/compute/kernels/aggregate_benchmark.cc b/cpp/src/arrow/compute/kernels/aggregate_benchmark.cc index 8522baaa24d6..c9edaaceb5ab 100644 --- a/cpp/src/arrow/compute/kernels/aggregate_benchmark.cc +++ b/cpp/src/arrow/compute/kernels/aggregate_benchmark.cc @@ -330,5 +330,85 @@ SUM_KERNEL_BENCHMARK(SumKernelInt16, Int16Type); SUM_KERNEL_BENCHMARK(SumKernelInt32, Int32Type); SUM_KERNEL_BENCHMARK(SumKernelInt64, Int64Type); +template +void ModeKernelBench(benchmark::State& state) { + using CType = typename TypeTraits::CType; + + RegressionArgs args(state); + const int64_t array_size = args.size / sizeof(CType); + auto rand = random::RandomArrayGenerator(1924); + auto array = rand.Numeric(array_size, -100, 100, args.null_proportion); + + for (auto _ : state) { + ABORT_NOT_OK(Mode(array).status()); + } +} + +template <> +void ModeKernelBench(benchmark::State& state) { + RegressionArgs args(state); + auto rand = random::RandomArrayGenerator(1924); + auto array = rand.Boolean(args.size * 8, 0.5, args.null_proportion); + + for (auto _ : state) { + ABORT_NOT_OK(Mode(array).status()); + } +} + +static void ModeKernelBenchArgs(benchmark::internal::Benchmark* bench) { + BenchmarkSetArgsWithSizes(bench, {1 * 1024 * 1024}); // 1M +} + +#define MODE_KERNEL_BENCHMARK(FuncName, Type) \ + static void FuncName(benchmark::State& state) { ModeKernelBench(state); } \ + BENCHMARK(FuncName)->Apply(ModeKernelBenchArgs) + +MODE_KERNEL_BENCHMARK(ModeKernelBoolean, BooleanType); +MODE_KERNEL_BENCHMARK(ModeKernelInt8, Int8Type); +MODE_KERNEL_BENCHMARK(ModeKernelInt16, Int16Type); +MODE_KERNEL_BENCHMARK(ModeKernelInt32, Int32Type); +MODE_KERNEL_BENCHMARK(ModeKernelInt64, Int64Type); + +template +static void MinMaxKernelBench(benchmark::State& state) { + using CType = typename TypeTraits::CType; + + RegressionArgs args(state); + const int64_t array_size = args.size / sizeof(CType); + auto rand = random::RandomArrayGenerator(1923); + auto array = rand.Numeric(array_size, -100, 100, args.null_proportion); + + for (auto _ : state) { + ABORT_NOT_OK(MinMax(array).status()); + } +} + +static void MinMaxKernelBenchArgs(benchmark::internal::Benchmark* bench) { + BenchmarkSetArgsWithSizes(bench, {1 * 1024 * 1024}); // 1M +} + +#define MINMAX_KERNEL_BENCHMARK(FuncName, Type) \ + static void FuncName(benchmark::State& state) { MinMaxKernelBench(state); } \ + BENCHMARK(FuncName)->Apply(MinMaxKernelBenchArgs) + +MINMAX_KERNEL_BENCHMARK(MinMaxKernelFloat, FloatType); +MINMAX_KERNEL_BENCHMARK(MinMaxKernelDouble, DoubleType); +MINMAX_KERNEL_BENCHMARK(MinMaxKernelInt8, Int8Type); +MINMAX_KERNEL_BENCHMARK(MinMaxKernelInt16, Int16Type); +MINMAX_KERNEL_BENCHMARK(MinMaxKernelInt32, Int32Type); +MINMAX_KERNEL_BENCHMARK(MinMaxKernelInt64, Int64Type); + +static void CountKernelBenchInt64(benchmark::State& state) { + RegressionArgs args(state); + const int64_t array_size = args.size / sizeof(int64_t); + auto rand = random::RandomArrayGenerator(1923); + auto array = rand.Numeric(array_size, -100, 100, args.null_proportion); + + for (auto _ : state) { + ABORT_NOT_OK(Count(array->Slice(1, array_size)).status()); + } +} +BENCHMARK(CountKernelBenchInt64)->Args({1 * 1024 * 1024, 2}); // 1M with 50% null. + } // namespace compute } // namespace arrow diff --git a/cpp/src/arrow/compute/kernels/aggregate_mode.cc b/cpp/src/arrow/compute/kernels/aggregate_mode.cc new file mode 100644 index 000000000000..11aef51564fe --- /dev/null +++ b/cpp/src/arrow/compute/kernels/aggregate_mode.cc @@ -0,0 +1,282 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include +#include + +#include "arrow/compute/kernels/aggregate_basic_internal.h" + +namespace arrow { +namespace compute { +namespace aggregate { + +namespace { + +// {value:count} map +template +using CounterMap = std::unordered_map; + +// map based counter for floating points +template +enable_if_t::value, CounterMap> CountValuesByMap( + const ArrayType& array, int64_t& nan_count) { + CounterMap value_counts_map; + + nan_count = 0; + if (array.length() > array.null_count()) { + VisitArrayDataInline( + *array.data(), + [&](CType value) { + if (std::isnan(value)) { + ++nan_count; + } else { + ++value_counts_map[value]; + } + }, + []() {}); + } + + return value_counts_map; +} + +// map base counter for non floating points +template +enable_if_t::value, CounterMap> CountValuesByMap( + const ArrayType& array) { + CounterMap value_counts_map; + + if (array.length() > array.null_count()) { + VisitArrayDataInline( + *array.data(), [&](CType value) { ++value_counts_map[value]; }, []() {}); + } + + return value_counts_map; +} + +// vector based counter for bool/int8 or integers with small value range +template +CounterMap CountValuesByVector(const ArrayType& array, CType min, CType max) { + const int range = static_cast(max - min); + DCHECK(range >= 0 && range < 64 * 1024 * 1024); + + std::vector value_counts_vector(range + 1); + if (array.length() > array.null_count()) { + VisitArrayDataInline( + *array.data(), [&](CType value) { ++value_counts_vector[value - min]; }, []() {}); + } + + // Transfer value counts to a map to be consistent with other chunks + CounterMap value_counts_map(range + 1); + for (int i = 0; i <= range; ++i) { + CType value = static_cast(i + min); + int64_t count = value_counts_vector[i]; + if (count) { + value_counts_map[value] = count; + } + } + + return value_counts_map; +} + +// map or vector based counter for int16/32/64 per value range +template +CounterMap CountValuesByMapOrVector(const ArrayType& array) { + // see https://issues.apache.org/jira/browse/ARROW-9873 + static constexpr int kMinArraySize = 8192 / sizeof(CType); + static constexpr int kMaxValueRange = 16384; + + if ((array.length() - array.null_count()) >= kMinArraySize) { + CType min = std::numeric_limits::max(); + CType max = std::numeric_limits::min(); + + VisitArrayDataInline( + *array.data(), + [&](CType value) { + min = std::min(min, value); + max = std::max(max, value); + }, + []() {}); + + if (static_cast(max) - static_cast(min) <= kMaxValueRange) { + return CountValuesByVector(array, min, max); + } + } + return CountValuesByMap(array); +} + +// bool, int8 +template +enable_if_t::value && sizeof(CType) == 1, CounterMap> +CountValues(const ArrayType& array, int64_t& nan_count) { + using Limits = std::numeric_limits; + nan_count = 0; + return CountValuesByVector(array, Limits::min(), Limits::max()); +} + +// int16/32/64 +template +enable_if_t::value && (sizeof(CType) > 1), CounterMap> +CountValues(const ArrayType& array, int64_t& nan_count) { + nan_count = 0; + return CountValuesByMapOrVector(array); +} + +// float/double +template +enable_if_t<(std::is_floating_point::value), CounterMap> // NOLINT format +CountValues(const ArrayType& array, int64_t& nan_count) { + nan_count = 0; + return CountValuesByMap(array, nan_count); +} + +template +struct ModeState { + using ThisType = ModeState; + using CType = typename ArrowType::c_type; + + void MergeFrom(ThisType&& state) { + if (this->value_counts.empty()) { + this->value_counts = std::move(state.value_counts); + } else { + for (const auto& value_count : state.value_counts) { + auto value = value_count.first; + auto count = value_count.second; + this->value_counts[value] += count; + } + } + if (is_floating_type::value) { + this->nan_count += state.nan_count; + } + } + + std::pair Finalize() { + CType mode = std::numeric_limits::min(); + int64_t count = 0; + + for (const auto& value_count : this->value_counts) { + auto this_value = value_count.first; + auto this_count = value_count.second; + if (this_count > count || (this_count == count && this_value < mode)) { + count = this_count; + mode = this_value; + } + } + if (is_floating_type::value && this->nan_count > count) { + count = this->nan_count; + mode = static_cast(NAN); + } + return std::make_pair(mode, count); + } + + int64_t nan_count = 0; // only make sense to floating types + CounterMap value_counts; +}; + +template +struct ModeImpl : public ScalarAggregator { + using ThisType = ModeImpl; + using ArrayType = typename TypeTraits::ArrayType; + + explicit ModeImpl(const std::shared_ptr& out_type) : out_type(out_type) {} + + void Consume(KernelContext*, const ExecBatch& batch) override { + ArrayType array(batch[0].array()); + this->state.value_counts = CountValues(array, this->state.nan_count); + } + + void MergeFrom(KernelContext*, KernelState&& src) override { + auto& other = checked_cast(src); + this->state.MergeFrom(std::move(other.state)); + } + + void Finalize(KernelContext*, Datum* out) override { + using ModeType = typename TypeTraits::ScalarType; + using CountType = typename TypeTraits::ScalarType; + + std::vector> values; + auto mode_count = this->state.Finalize(); + auto mode = mode_count.first; + auto count = mode_count.second; + if (count == 0) { + values = {std::make_shared(), std::make_shared()}; + } else { + values = {std::make_shared(mode), std::make_shared(count)}; + } + out->value = std::make_shared(std::move(values), this->out_type); + } + + std::shared_ptr out_type; + ModeState state; +}; + +struct ModeInitState { + std::unique_ptr state; + KernelContext* ctx; + const DataType& in_type; + const std::shared_ptr& out_type; + + ModeInitState(KernelContext* ctx, const DataType& in_type, + const std::shared_ptr& out_type) + : ctx(ctx), in_type(in_type), out_type(out_type) {} + + Status Visit(const DataType&) { return Status::NotImplemented("No mode implemented"); } + + Status Visit(const HalfFloatType&) { + return Status::NotImplemented("No mode implemented"); + } + + template + enable_if_t::value || is_boolean_type::value, Status> Visit( + const Type&) { + state.reset(new ModeImpl(out_type)); + return Status::OK(); + } + + std::unique_ptr Create() { + ctx->SetStatus(VisitTypeInline(in_type, this)); + return std::move(state); + } +}; + +std::unique_ptr ModeInit(KernelContext* ctx, const KernelInitArgs& args) { + ModeInitState visitor(ctx, *args.inputs[0].type, + args.kernel->signature->out_type().type()); + return visitor.Create(); +} + +void AddModeKernels(KernelInit init, const std::vector>& types, + ScalarAggregateFunction* func) { + for (const auto& ty : types) { + // array[T] -> scalar[struct] + auto out_ty = struct_({field("mode", ty), field("count", int64())}); + auto sig = KernelSignature::Make({InputType::Array(ty)}, ValueDescr::Scalar(out_ty)); + AddAggKernel(std::move(sig), init, func); + } +} + +} // namespace + +std::shared_ptr AddModeAggKernels() { + auto func = std::make_shared("mode", Arity::Unary()); + AddModeKernels(ModeInit, {boolean()}, func.get()); + AddModeKernels(ModeInit, internal::NumericTypes(), func.get()); + return func; +} + +} // namespace aggregate +} // namespace compute +} // namespace arrow diff --git a/cpp/src/arrow/compute/kernels/aggregate_test.cc b/cpp/src/arrow/compute/kernels/aggregate_test.cc index ad669b29c4b6..39b3f8827fb1 100644 --- a/cpp/src/arrow/compute/kernels/aggregate_test.cc +++ b/cpp/src/arrow/compute/kernels/aggregate_test.cc @@ -19,6 +19,7 @@ #include #include #include +#include #include #include @@ -43,9 +44,9 @@ using internal::checked_pointer_cast; namespace compute { -/// -/// Sum -/// +// +// Sum +// template using SumResult = @@ -262,9 +263,9 @@ TYPED_TEST(TestRandomNumericSumKernel, RandomSliceArraySum) { } } -/// -/// Count -/// +// +// Count +// using CountPair = std::pair; @@ -325,9 +326,9 @@ TYPED_TEST(TestRandomNumericCountKernel, RandomArrayCount) { } } -/// -/// Mean -/// +// +// Mean +// template static Datum NaiveMean(const Array& array) { @@ -425,9 +426,9 @@ TYPED_TEST(TestRandomNumericMeanKernel, RandomArrayMeanOverflow) { } } -/// -/// Min / Max -/// +// +// Min / Max +// template class TestPrimitiveMinMaxKernel : public ::testing::Test { @@ -512,7 +513,7 @@ TEST_F(TestBooleanMinMaxKernel, Basics) { this->AssertMinMaxIs(chunked_input2, false, false, options); this->AssertMinMaxIs(chunked_input3, false, true, options); - options = MinMaxOptions(MinMaxOptions::OUTPUT_NULL); + options = MinMaxOptions(MinMaxOptions::EMIT_NULL); this->AssertMinMaxIsNull("[]", options); this->AssertMinMaxIsNull("[null, null, null]", options); this->AssertMinMaxIsNull("[false, null, false]", options); @@ -542,7 +543,7 @@ TYPED_TEST(TestIntegerMinMaxKernel, Basics) { this->AssertMinMaxIs(chunked_input2, 1, 9, options); this->AssertMinMaxIs(chunked_input3, 1, 9, options); - options = MinMaxOptions(MinMaxOptions::OUTPUT_NULL); + options = MinMaxOptions(MinMaxOptions::EMIT_NULL); this->AssertMinMaxIs("[5, 1, 2, 3, 4]", 1, 5, options); // output null this->AssertMinMaxIsNull("[5, null, 2, 3, 4]", options); @@ -569,7 +570,7 @@ TYPED_TEST(TestFloatingMinMaxKernel, Floats) { this->AssertMinMaxIs(chunked_input2, 1, 9, options); this->AssertMinMaxIs(chunked_input3, 1, 9, options); - options = MinMaxOptions(MinMaxOptions::OUTPUT_NULL); + options = MinMaxOptions(MinMaxOptions::EMIT_NULL); this->AssertMinMaxIs("[5, 1, 2, 3, 4]", 1, 5, options); this->AssertMinMaxIs("[5, -Inf, 2, 3, 4]", -INFINITY, 5, options); // output null @@ -594,5 +595,530 @@ TYPED_TEST(TestFloatingMinMaxKernel, DefaultOptions) { AssertDatumsEqual(explicit_defaults, no_options_provided); } +template +struct MinMaxResult { + using T = typename ArrowType::c_type; + + T min = 0; + T max = 0; + bool is_valid = false; +}; + +template +static enable_if_integer> NaiveMinMax( + const Array& array) { + using T = typename ArrowType::c_type; + using ArrayType = typename TypeTraits::ArrayType; + + MinMaxResult result; + + const auto& array_numeric = reinterpret_cast(array); + const auto values = array_numeric.raw_values(); + + if (array.length() <= array.null_count()) { // All null values + return result; + } + + T min = std::numeric_limits::max(); + T max = std::numeric_limits::min(); + if (array.null_count() != 0) { // Some values are null + internal::BitmapReader reader(array.null_bitmap_data(), array.offset(), + array.length()); + for (int64_t i = 0; i < array.length(); i++) { + if (reader.IsSet()) { + min = std::min(min, values[i]); + max = std::max(max, values[i]); + } + reader.Next(); + } + } else { // All true values + for (int64_t i = 0; i < array.length(); i++) { + min = std::min(min, values[i]); + max = std::max(max, values[i]); + } + } + + result.min = min; + result.max = max; + result.is_valid = true; + return result; +} + +template +static enable_if_floating_point> NaiveMinMax( + const Array& array) { + using T = typename ArrowType::c_type; + using ArrayType = typename TypeTraits::ArrayType; + + MinMaxResult result; + + const auto& array_numeric = reinterpret_cast(array); + const auto values = array_numeric.raw_values(); + + if (array.length() <= array.null_count()) { // All null values + return result; + } + + T min = std::numeric_limits::infinity(); + T max = -std::numeric_limits::infinity(); + if (array.null_count() != 0) { // Some values are null + internal::BitmapReader reader(array.null_bitmap_data(), array.offset(), + array.length()); + for (int64_t i = 0; i < array.length(); i++) { + if (reader.IsSet()) { + min = std::fmin(min, values[i]); + max = std::fmax(max, values[i]); + } + reader.Next(); + } + } else { // All true values + for (int64_t i = 0; i < array.length(); i++) { + min = std::fmin(min, values[i]); + max = std::fmax(max, values[i]); + } + } + + result.min = min; + result.max = max; + result.is_valid = true; + return result; +} + +template +void ValidateMinMax(const Array& array) { + using Traits = TypeTraits; + using ScalarType = typename Traits::ScalarType; + + ASSERT_OK_AND_ASSIGN(Datum out, MinMax(array)); + const StructScalar& value = out.scalar_as(); + + auto expected = NaiveMinMax(array); + const auto& out_min = checked_cast(*value.value[0]); + const auto& out_max = checked_cast(*value.value[1]); + + if (expected.is_valid) { + ASSERT_TRUE(out_min.is_valid); + ASSERT_TRUE(out_max.is_valid); + ASSERT_EQ(expected.min, out_min.value); + ASSERT_EQ(expected.max, out_max.value); + } else { // All null values + ASSERT_FALSE(out_min.is_valid); + ASSERT_FALSE(out_max.is_valid); + } +} + +template +class TestRandomNumericMinMaxKernel : public ::testing::Test {}; + +TYPED_TEST_SUITE(TestRandomNumericMinMaxKernel, NumericArrowTypes); +TYPED_TEST(TestRandomNumericMinMaxKernel, RandomArrayMinMax) { + auto rand = random::RandomArrayGenerator(0x8afc055); + // Test size up to 1<<11 (2048). + for (size_t i = 3; i < 12; i += 2) { + for (auto null_probability : {0.0, 0.01, 0.1, 0.5, 0.99, 1.0}) { + int64_t base_length = (1UL << i) + 2; + auto array = rand.Numeric(base_length, 0, 100, null_probability); + for (auto length_adjust : {-2, -1, 0, 1, 2}) { + int64_t length = (1UL << i) + length_adjust; + ValidateMinMax(*array->Slice(0, length)); + } + } + } +} + +// +// Mode +// + +template +class TestPrimitiveModeKernel : public ::testing::Test { + public: + using ArrowType = T; + using Traits = TypeTraits; + using c_type = typename ArrowType::c_type; + using ModeType = typename Traits::ScalarType; + using CountType = typename TypeTraits::ScalarType; + + void AssertModeIs(const Datum& array, c_type expected_mode, int64_t expected_count) { + ASSERT_OK_AND_ASSIGN(Datum out, Mode(array)); + const StructScalar& value = out.scalar_as(); + + const auto& out_mode = checked_cast(*value.value[0]); + ASSERT_EQ(expected_mode, out_mode.value); + + const auto& out_count = checked_cast(*value.value[1]); + ASSERT_EQ(expected_count, out_count.value); + } + + void AssertModeIs(const std::string& json, c_type expected_mode, + int64_t expected_count) { + auto array = ArrayFromJSON(type_singleton(), json); + AssertModeIs(array, expected_mode, expected_count); + } + + void AssertModeIs(const std::vector& json, c_type expected_mode, + int64_t expected_count) { + auto chunked = ChunkedArrayFromJSON(type_singleton(), json); + AssertModeIs(chunked, expected_mode, expected_count); + } + + void AssertModeIsNull(const Datum& array) { + ASSERT_OK_AND_ASSIGN(Datum out, Mode(array)); + const StructScalar& value = out.scalar_as(); + + for (const auto& val : value.value) { + ASSERT_FALSE(val->is_valid); + } + } + + void AssertModeIsNull(const std::string& json) { + auto array = ArrayFromJSON(type_singleton(), json); + AssertModeIsNull(array); + } + + void AssertModeIsNull(const std::vector& json) { + auto chunked = ChunkedArrayFromJSON(type_singleton(), json); + AssertModeIsNull(chunked); + } + + void AssertModeIsNaN(const Datum& array, int64_t expected_count) { + ASSERT_OK_AND_ASSIGN(Datum out, Mode(array)); + const StructScalar& value = out.scalar_as(); + + const auto& out_mode = checked_cast(*value.value[0]); + ASSERT_NE(out_mode.value, out_mode.value); // NaN != NaN + + const auto& out_count = checked_cast(*value.value[1]); + ASSERT_EQ(expected_count, out_count.value); + } + + void AssertModeIsNaN(const std::string& json, int64_t expected_count) { + auto array = ArrayFromJSON(type_singleton(), json); + AssertModeIsNaN(array, expected_count); + } + + void AssertModeIsNaN(const std::vector& json, int64_t expected_count) { + auto chunked = ChunkedArrayFromJSON(type_singleton(), json); + AssertModeIsNaN(chunked, expected_count); + } + + std::shared_ptr type_singleton() { return Traits::type_singleton(); } +}; + +template +class TestIntegerModeKernel : public TestPrimitiveModeKernel {}; + +template +class TestFloatingModeKernel : public TestPrimitiveModeKernel {}; + +class TestBooleanModeKernel : public TestPrimitiveModeKernel {}; + +class TestInt8ModeKernelValueRange : public TestPrimitiveModeKernel {}; + +class TestInt32ModeKernel : public TestPrimitiveModeKernel {}; + +TEST_F(TestBooleanModeKernel, Basics) { + this->AssertModeIs("[false, false]", false, 2); + this->AssertModeIs("[false, false, true, true, true]", true, 3); + this->AssertModeIs("[true, false, false, true, true]", true, 3); + this->AssertModeIs("[false, false, true, true, true, false]", false, 3); + + this->AssertModeIs("[true, null, false, false, null, true, null, null, true]", true, 3); + this->AssertModeIsNull("[null, null, null]"); + this->AssertModeIsNull("[]"); + + this->AssertModeIs({"[true, false]", "[true, true]", "[false, false]"}, false, 3); + this->AssertModeIs({"[true, null]", "[]", "[null, false]"}, false, 1); + this->AssertModeIsNull({"[null, null]", "[]", "[null]"}); +} + +TYPED_TEST_SUITE(TestIntegerModeKernel, IntegralArrowTypes); +TYPED_TEST(TestIntegerModeKernel, Basics) { + this->AssertModeIs("[5, 1, 1, 5, 5]", 5, 3); + this->AssertModeIs("[5, 1, 1, 5, 5, 1]", 1, 3); + this->AssertModeIs("[127, 0, 127, 127, 0, 1, 0, 127]", 127, 4); + + this->AssertModeIs("[null, null, 2, null, 1]", 1, 1); + this->AssertModeIsNull("[null, null, null]"); + this->AssertModeIsNull("[]"); + + this->AssertModeIs({"[5]", "[1, 1, 5]", "[5]"}, 5, 3); + this->AssertModeIs({"[5]", "[1, 1, 5]", "[5, 1]"}, 1, 3); + this->AssertModeIsNull({"[null, null]", "[]", "[null]"}); +} + +TYPED_TEST_SUITE(TestFloatingModeKernel, RealArrowTypes); +TYPED_TEST(TestFloatingModeKernel, Floats) { + this->AssertModeIs("[5, 1, 1, 5, 5]", 5, 3); + this->AssertModeIs("[5, 1, 1, 5, 5, 1]", 1, 3); + this->AssertModeIs("[Inf, 100, Inf, 100, Inf]", INFINITY, 3); + this->AssertModeIs("[Inf, -Inf, Inf, -Inf]", -INFINITY, 2); + + this->AssertModeIs("[null, null, 2, null, 1]", 1, 1); + this->AssertModeIs("[NaN, NaN, 1, null, 1]", 1, 2); + + this->AssertModeIsNull("[null, null, null]"); + this->AssertModeIsNull("[]"); + + this->AssertModeIsNaN("[NaN, NaN, 1]", 2); + this->AssertModeIsNaN("[NaN, NaN, null]", 2); + this->AssertModeIsNaN("[NaN, NaN, NaN]", 3); + + this->AssertModeIs({"[Inf, 100]", "[Inf, 100]", "[Inf]"}, INFINITY, 3); + this->AssertModeIsNull({"[null, null]", "[]", "[null]"}); + this->AssertModeIsNaN({"[NaN, 1]", "[NaN, 1]", "[NaN]"}, 3); +} + +TEST_F(TestInt8ModeKernelValueRange, Basics) { + this->AssertModeIs("[0, 127, -128, -128]", -128, 2); + this->AssertModeIs("[127, 127, 127]", 127, 3); +} + +template +struct ModeResult { + using T = typename ArrowType::c_type; + + T mode = std::numeric_limits::min(); + int64_t count = 0; +}; + +template +ModeResult NaiveMode(const Array& array) { + using ArrayType = typename TypeTraits::ArrayType; + using CTYPE = typename ArrowType::c_type; + + std::unordered_map value_counts; + + const auto& array_numeric = reinterpret_cast(array); + const auto values = array_numeric.raw_values(); + internal::BitmapReader reader(array.null_bitmap_data(), array.offset(), array.length()); + for (int64_t i = 0; i < array.length(); ++i) { + if (reader.IsSet()) { + ++value_counts[values[i]]; + } + reader.Next(); + } + + ModeResult result; + for (const auto& value_count : value_counts) { + auto value = value_count.first; + auto count = value_count.second; + if (count > result.count || (count == result.count && value < result.mode)) { + result.count = count; + result.mode = value; + } + } + + return result; +} + +template +void CheckModeWithRange(CTYPE range_min, CTYPE range_max) { + using ModeScalar = typename TypeTraits::ScalarType; + using CountScalar = typename TypeTraits::ScalarType; + + auto rand = random::RandomArrayGenerator(0x5487655); + // 32K items (>= counting mode cutoff) within range, 10% null + auto array = rand.Numeric(32 * 1024, range_min, range_max, 0.1); + + auto expected = NaiveMode(*array); + ASSERT_OK_AND_ASSIGN(Datum out, Mode(array)); + const StructScalar& value = out.scalar_as(); + + ASSERT_TRUE(value.is_valid); + const auto& out_mode = checked_cast(*value.value[0]); + const auto& out_count = checked_cast(*value.value[1]); + ASSERT_EQ(out_mode.value, expected.mode); + ASSERT_EQ(out_count.value, expected.count); +} + +TEST_F(TestInt32ModeKernel, SmallValueRange) { + // Small value range => should exercise counter-based Mode implementation + CheckModeWithRange(-100, 100); +} + +TEST_F(TestInt32ModeKernel, LargeValueRange) { + // Large value range => should exercise hashmap-based Mode implementation + CheckModeWithRange(-10000000, 10000000); +} + +// +// Variance/Stddev +// + +template +class TestPrimitiveVarStdKernel : public ::testing::Test { + public: + using Traits = TypeTraits; + using ScalarType = typename TypeTraits::ScalarType; + + void AssertVarStdIs(const Array& array, const VarianceOptions& options, + double expected_var, double diff = 0) { + AssertVarStdIsInternal(array, options, expected_var, diff); + } + + void AssertVarStdIs(const std::shared_ptr& array, + const VarianceOptions& options, double expected_var, + double diff = 0) { + AssertVarStdIsInternal(array, options, expected_var, diff); + } + + void AssertVarStdIs(const std::string& json, const VarianceOptions& options, + double expected_var) { + auto array = ArrayFromJSON(type_singleton(), json); + AssertVarStdIs(*array, options, expected_var); + } + + void AssertVarStdIs(const std::vector& json, + const VarianceOptions& options, double expected_var) { + auto chunked = ChunkedArrayFromJSON(type_singleton(), json); + AssertVarStdIs(chunked, options, expected_var); + } + + void AssertVarStdIsInvalid(const Array& array, const VarianceOptions& options) { + AssertVarStdIsInvalidInternal(array, options); + } + + void AssertVarStdIsInvalid(const std::shared_ptr& array, + const VarianceOptions& options) { + AssertVarStdIsInvalidInternal(array, options); + } + + void AssertVarStdIsInvalid(const std::string& json, const VarianceOptions& options) { + auto array = ArrayFromJSON(type_singleton(), json); + AssertVarStdIsInvalid(*array, options); + } + + void AssertVarStdIsInvalid(const std::vector& json, + const VarianceOptions& options) { + auto array = ChunkedArrayFromJSON(type_singleton(), json); + AssertVarStdIsInvalid(array, options); + } + + std::shared_ptr type_singleton() { return Traits::type_singleton(); } + + private: + void AssertVarStdIsInternal(const Datum& array, const VarianceOptions& options, + double expected_var, double diff = 0) { + ASSERT_OK_AND_ASSIGN(Datum out_var, Variance(array, options)); + ASSERT_OK_AND_ASSIGN(Datum out_std, Stddev(array, options)); + auto var = checked_cast(out_var.scalar().get()); + auto std = checked_cast(out_std.scalar().get()); + ASSERT_TRUE(var->is_valid && std->is_valid); + ASSERT_DOUBLE_EQ(std->value * std->value, var->value); + if (diff == 0) { + ASSERT_DOUBLE_EQ(var->value, expected_var); // < 4ULP + } else { + ASSERT_NEAR(var->value, expected_var, diff); + } + } + + void AssertVarStdIsInvalidInternal(const Datum& array, const VarianceOptions& options) { + ASSERT_OK_AND_ASSIGN(Datum out_var, Variance(array, options)); + ASSERT_OK_AND_ASSIGN(Datum out_std, Stddev(array, options)); + auto var = checked_cast(out_var.scalar().get()); + auto std = checked_cast(out_std.scalar().get()); + ASSERT_FALSE(var->is_valid || std->is_valid); + } +}; + +template +class TestNumericVarStdKernel : public TestPrimitiveVarStdKernel {}; + +// Reference value from numpy.var +TYPED_TEST_SUITE(TestNumericVarStdKernel, NumericArrowTypes); +TYPED_TEST(TestNumericVarStdKernel, Basics) { + VarianceOptions options; // ddof = 0, population variance/stddev + + this->AssertVarStdIs("[100]", options, 0); + this->AssertVarStdIs("[1, 2, 3]", options, 0.6666666666666666); + this->AssertVarStdIs("[null, 1, 2, null, 3]", options, 0.6666666666666666); + + std::vector chunks; + chunks = {"[]", "[1]", "[2]", "[null]", "[3]"}; + this->AssertVarStdIs(chunks, options, 0.6666666666666666); + chunks = {"[1, 2, 3]", "[4, 5, 6]", "[7, 8]"}; + this->AssertVarStdIs(chunks, options, 5.25); + chunks = {"[1, 2, 3, 4, 5, 6, 7]", "[8]"}; + this->AssertVarStdIs(chunks, options, 5.25); + + this->AssertVarStdIsInvalid("[null, null, null]", options); + this->AssertVarStdIsInvalid("[]", options); + this->AssertVarStdIsInvalid("[]", options); + + options.ddof = 1; // sample variance/stddev + + this->AssertVarStdIs("[1, 2]", options, 0.5); + + chunks = {"[1]", "[2]"}; + this->AssertVarStdIs(chunks, options, 0.5); + chunks = {"[1, 2, 3]", "[4, 5, 6]", "[7, 8]"}; + this->AssertVarStdIs(chunks, options, 6.0); + chunks = {"[1, 2, 3, 4, 5, 6, 7]", "[8]"}; + this->AssertVarStdIs(chunks, options, 6.0); + + this->AssertVarStdIsInvalid("[100]", options); + this->AssertVarStdIsInvalid("[100, null, null]", options); + chunks = {"[100]", "[null]", "[]"}; + this->AssertVarStdIsInvalid(chunks, options); +} + +class TestVarStdKernelStability : public TestPrimitiveVarStdKernel {}; + +// Test numerical stability +TEST_F(TestVarStdKernelStability, Basics) { + VarianceOptions options{1}; // ddof = 1 + this->AssertVarStdIs("[100000004, 100000007, 100000013, 100000016]", options, 30.0); + this->AssertVarStdIs("[1000000004, 1000000007, 1000000013, 1000000016]", options, 30.0); +} + +// Calculate reference variance with Welford's online algorithm +// https://en.wikipedia.org/wiki/Algorithms_for_calculating_variance#Welford's_online_algorithm +std::pair WelfordVar(const Array& array) { + const auto& array_numeric = reinterpret_cast(array); + const auto values = array_numeric.raw_values(); + internal::BitmapReader reader(array.null_bitmap_data(), array.offset(), array.length()); + double count = 0, mean = 0, m2 = 0; + for (int64_t i = 0; i < array.length(); ++i) { + if (reader.IsSet()) { + ++count; + double delta = values[i] - mean; + mean += delta / count; + double delta2 = values[i] - mean; + m2 += delta * delta2; + } + reader.Next(); + } + return std::make_pair(m2 / count, m2 / (count - 1)); +} + +class TestVarStdKernelRandom : public TestPrimitiveVarStdKernel {}; + +TEST_F(TestVarStdKernelRandom, Basics) { + // Cut array into small chunks + constexpr int array_size = 5000; + constexpr int chunk_size_max = 50; + constexpr int chunk_count = array_size / chunk_size_max; + + auto rand = random::RandomArrayGenerator(0x5487656); + auto array = rand.Numeric(array_size, -10000.0, 100000.0, 0.1); + auto chunk_size_array = rand.Numeric(chunk_count, 0, chunk_size_max); + const int* chunk_size = chunk_size_array->data()->GetValues(1); + int total_size = 0; + + ArrayVector array_vector; + for (int i = 0; i < chunk_count; ++i) { + array_vector.emplace_back(array->Slice(total_size, chunk_size[i])); + total_size += chunk_size[i]; + } + auto chunked = *ChunkedArray::Make(array_vector); + + double var_population, var_sample; + std::tie(var_population, var_sample) = WelfordVar(*(array->Slice(0, total_size))); + + this->AssertVarStdIs(chunked, VarianceOptions{0}, var_population, 0.0001); + this->AssertVarStdIs(chunked, VarianceOptions{1}, var_sample, 0.0001); +} + } // namespace compute } // namespace arrow diff --git a/cpp/src/arrow/compute/kernels/aggregate_var_std.cc b/cpp/src/arrow/compute/kernels/aggregate_var_std.cc new file mode 100644 index 000000000000..e2b98bb38fc7 --- /dev/null +++ b/cpp/src/arrow/compute/kernels/aggregate_var_std.cc @@ -0,0 +1,202 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "arrow/compute/kernels/aggregate_basic_internal.h" + +namespace arrow { +namespace compute { +namespace aggregate { + +namespace { + +template +struct VarStdState { + using ArrayType = typename TypeTraits::ArrayType; + using c_type = typename ArrowType::c_type; + using ThisType = VarStdState; + + // Calculate `m2` (sum((X-mean)^2)) of one chunk with `two pass algorithm` + // https://en.wikipedia.org/wiki/Algorithms_for_calculating_variance#Two-pass_algorithm + // Always use `double` to calculate variance for any array type + void Consume(const ArrayType& array) { + int64_t count = array.length() - array.null_count(); + if (count == 0) { + return; + } + + double sum = 0; + VisitArrayDataInline( + *array.data(), [&sum](c_type value) { sum += static_cast(value); }, + []() {}); + + double mean = sum / count, m2 = 0; + VisitArrayDataInline( + *array.data(), + [mean, &m2](c_type value) { + double v = static_cast(value); + m2 += (v - mean) * (v - mean); + }, + []() {}); + + this->count = count; + this->sum = sum; + this->m2 = m2; + } + + // Combine `m2` from two chunks + // https://en.wikipedia.org/wiki/Algorithms_for_calculating_variance#Parallel_algorithm + void MergeFrom(const ThisType& state) { + if (state.count == 0) { + return; + } + if (this->count == 0) { + this->count = state.count; + this->sum = state.sum; + this->m2 = state.m2; + return; + } + double delta = this->sum / this->count - state.sum / state.count; + this->m2 += state.m2 + + delta * delta * this->count * state.count / (this->count + state.count); + this->count += state.count; + this->sum += state.sum; + } + + int64_t count = 0; + double sum = 0; + double m2 = 0; // sum((X-mean)^2) +}; + +enum class VarOrStd : bool { Var, Std }; + +template +struct VarStdImpl : public ScalarAggregator { + using ThisType = VarStdImpl; + using ArrayType = typename TypeTraits::ArrayType; + + explicit VarStdImpl(const std::shared_ptr& out_type, + const VarianceOptions& options, VarOrStd return_type) + : out_type(out_type), options(options), return_type(return_type) {} + + void Consume(KernelContext*, const ExecBatch& batch) override { + ArrayType array(batch[0].array()); + this->state.Consume(array); + } + + void MergeFrom(KernelContext*, KernelState&& src) override { + const auto& other = checked_cast(src); + this->state.MergeFrom(other.state); + } + + void Finalize(KernelContext*, Datum* out) override { + if (this->state.count <= options.ddof) { + out->value = std::make_shared(); + } else { + double var = this->state.m2 / (this->state.count - options.ddof); + out->value = + std::make_shared(return_type == VarOrStd::Var ? var : sqrt(var)); + } + } + + std::shared_ptr out_type; + VarStdState state; + VarianceOptions options; + VarOrStd return_type; +}; + +struct VarStdInitState { + std::unique_ptr state; + KernelContext* ctx; + const DataType& in_type; + const std::shared_ptr& out_type; + const VarianceOptions& options; + VarOrStd return_type; + + VarStdInitState(KernelContext* ctx, const DataType& in_type, + const std::shared_ptr& out_type, + const VarianceOptions& options, VarOrStd return_type) + : ctx(ctx), + in_type(in_type), + out_type(out_type), + options(options), + return_type(return_type) {} + + Status Visit(const DataType&) { + return Status::NotImplemented("No variance/stddev implemented"); + } + + Status Visit(const HalfFloatType&) { + return Status::NotImplemented("No variance/stddev implemented"); + } + + template + enable_if_t::value, Status> Visit(const Type&) { + state.reset(new VarStdImpl(out_type, options, return_type)); + return Status::OK(); + } + + std::unique_ptr Create() { + ctx->SetStatus(VisitTypeInline(in_type, this)); + return std::move(state); + } +}; + +std::unique_ptr StddevInit(KernelContext* ctx, const KernelInitArgs& args) { + VarStdInitState visitor( + ctx, *args.inputs[0].type, args.kernel->signature->out_type().type(), + static_cast(*args.options), VarOrStd::Std); + return visitor.Create(); +} + +std::unique_ptr VarianceInit(KernelContext* ctx, + const KernelInitArgs& args) { + VarStdInitState visitor( + ctx, *args.inputs[0].type, args.kernel->signature->out_type().type(), + static_cast(*args.options), VarOrStd::Var); + return visitor.Create(); +} + +void AddVarStdKernels(KernelInit init, + const std::vector>& types, + ScalarAggregateFunction* func) { + for (const auto& ty : types) { + auto sig = KernelSignature::Make({InputType::Array(ty)}, float64()); + AddAggKernel(std::move(sig), init, func); + } +} + +} // namespace + +std::shared_ptr AddStddevAggKernels() { + static auto default_std_options = VarianceOptions::Defaults(); + auto func = std::make_shared("stddev", Arity::Unary(), + &default_std_options); + AddVarStdKernels(StddevInit, internal::NumericTypes(), func.get()); + return func; +} + +std::shared_ptr AddVarianceAggKernels() { + static auto default_var_options = VarianceOptions::Defaults(); + auto func = std::make_shared("variance", Arity::Unary(), + &default_var_options); + AddVarStdKernels(VarianceInit, internal::NumericTypes(), func.get()); + return func; +} + +} // namespace aggregate +} // namespace compute +} // namespace arrow diff --git a/cpp/src/arrow/compute/kernels/codegen_internal.h b/cpp/src/arrow/compute/kernels/codegen_internal.h index 4d8918ae773b..1f9402498574 100644 --- a/cpp/src/arrow/compute/kernels/codegen_internal.h +++ b/cpp/src/arrow/compute/kernels/codegen_internal.h @@ -35,6 +35,7 @@ #include "arrow/status.h" #include "arrow/type.h" #include "arrow/type_traits.h" +#include "arrow/util/bit_block_counter.h" #include "arrow/util/bit_util.h" #include "arrow/util/bitmap_generate.h" #include "arrow/util/bitmap_reader.h" @@ -50,10 +51,14 @@ namespace arrow { +using internal::BinaryBitBlockCounter; +using internal::BitBlockCount; using internal::BitmapReader; using internal::checked_cast; using internal::FirstTimeBitmapWriter; using internal::GenerateBitsUnrolled; +using internal::VisitBitBlocksVoid; +using internal::VisitTwoBitBlocksVoid; namespace compute { namespace internal { @@ -121,6 +126,57 @@ struct OptionsWrapper : public KernelState { OptionsType options; }; +// ---------------------------------------------------------------------- +// Input and output value type definitions + +template +struct GetViewType; + +template +struct GetViewType> { + using T = typename Type::c_type; + using PhysicalType = T; + + static T LogicalValue(PhysicalType value) { return value; } +}; + +template +struct GetViewType::value || + is_fixed_size_binary_type::value>> { + using T = util::string_view; + using PhysicalType = T; + + static T LogicalValue(PhysicalType value) { return value; } +}; + +template <> +struct GetViewType { + using T = Decimal128; + using PhysicalType = util::string_view; + + static T LogicalValue(PhysicalType value) { + return Decimal128(reinterpret_cast(value.data())); + } +}; + +template +struct GetOutputType; + +template +struct GetOutputType> { + using T = typename Type::c_type; +}; + +template +struct GetOutputType::value>> { + using T = std::string; +}; + +template <> +struct GetOutputType { + using T = Decimal128; +}; + // ---------------------------------------------------------------------- // Iteration / value access utilities @@ -128,6 +184,8 @@ template using enable_if_has_c_type_not_boolean = enable_if_t::value && !is_boolean_type::value, R>; +// Iterator over various input array types, yielding a GetViewType + template struct ArrayIterator; @@ -135,6 +193,7 @@ template struct ArrayIterator> { using T = typename Type::c_type; const T* values; + explicit ArrayIterator(const ArrayData& data) : values(data.GetValues(1)) {} T operator()() { return *values++; } }; @@ -142,6 +201,7 @@ struct ArrayIterator> { template struct ArrayIterator> { BitmapReader reader; + explicit ArrayIterator(const ArrayData& data) : reader(data.buffers[1]->data(), data.offset, data.length) {} bool operator()() { @@ -159,6 +219,7 @@ struct ArrayIterator> { offset_type cur_offset; const char* data; int64_t position; + explicit ArrayIterator(const ArrayData& arr) : arr(arr), offsets(reinterpret_cast(arr.buffers[1]->data()) + @@ -168,13 +229,34 @@ struct ArrayIterator> { position(0) {} util::string_view operator()() { - offset_type next_offset = offsets[position++ + 1]; + offset_type next_offset = offsets[++position]; auto result = util::string_view(data + cur_offset, next_offset - cur_offset); cur_offset = next_offset; return result; } }; +// Iterator over various output array types, taking a GetOutputType + +template +struct OutputArrayWriter; + +template +struct OutputArrayWriter> { + using T = typename Type::c_type; + T* values; + + explicit OutputArrayWriter(ArrayData* data) : values(data->GetMutableValues(1)) {} + + void Write(T value) { *values++ = value; } + + // Note that this doesn't write the null bitmap, which should be consistent + // with Write / WriteNull calls + void WriteNull() { *values++ = T{}; } +}; + +// (Un)box Scalar to / from C++ value + template struct UnboxScalar; @@ -201,43 +283,6 @@ struct UnboxScalar { } }; -template -struct GetViewType; - -template -struct GetViewType> { - using T = typename Type::c_type; -}; - -template -struct GetViewType::value || - is_fixed_size_binary_type::value>> { - using T = util::string_view; -}; - -template <> -struct GetViewType { - using T = Decimal128; -}; - -template -struct GetOutputType; - -template -struct GetOutputType> { - using T = typename Type::c_type; -}; - -template -struct GetOutputType::value>> { - using T = std::string; -}; - -template <> -struct GetOutputType { - using T = Decimal128; -}; - template struct BoxScalar; @@ -264,6 +309,41 @@ struct BoxScalar { static void Box(T val, Scalar* out) { checked_cast(out)->value = val; } }; +// A VisitArrayDataInline variant that calls its visitor function with logical +// values, such as Decimal128 rather than util::string_view. + +template +static void VisitArrayValuesInline(const ArrayData& arr, VisitFunc&& valid_func, + NullFunc&& null_func) { + VisitArrayDataInline( + arr, + [&](typename GetViewType::PhysicalType v) { + valid_func(GetViewType::LogicalValue(std::move(v))); + }, + std::forward(null_func)); +} + +// Like VisitArrayValuesInline, but for binary functions. + +template +static void VisitTwoArrayValuesInline(const ArrayData& arr0, const ArrayData& arr1, + VisitFunc&& valid_func, NullFunc&& null_func) { + ArrayIterator arr0_it(arr0); + ArrayIterator arr1_it(arr1); + + auto visit_valid = [&](int64_t i) { + valid_func(GetViewType::LogicalValue(arr0_it()), + GetViewType::LogicalValue(arr1_it())); + }; + auto visit_null = [&]() { + arr0_it(); + arr1_it(); + null_func(); + }; + VisitTwoBitBlocksVoid(arr0.buffers[0], arr0.offset, arr1.buffers[0], arr1.offset, + arr0.length, std::move(visit_valid), std::move(visit_null)); +} + // ---------------------------------------------------------------------- // Reusable type resolvers @@ -406,26 +486,27 @@ struct OutputAdapter> { // The "Op" functor should have the form // // struct Op { -// template -// static OUT Call(KernelContext* ctx, ARG0 val) { +// template +// static OutValue Call(KernelContext* ctx, Arg0Value val) { // // implementation // } // }; template struct ScalarUnary { - using OUT = typename GetOutputType::T; - using ARG0 = typename GetViewType::T; + using OutValue = typename GetOutputType::T; + using Arg0Value = typename GetViewType::T; static void Array(KernelContext* ctx, const ArrayData& arg0, Datum* out) { ArrayIterator arg0_it(arg0); - OutputAdapter::Write( - ctx, out, [&]() -> OUT { return Op::template Call(ctx, arg0_it()); }); + OutputAdapter::Write(ctx, out, [&]() -> OutValue { + return Op::template Call(ctx, arg0_it()); + }); } static void Scalar(KernelContext* ctx, const Scalar& arg0, Datum* out) { if (arg0.is_valid) { - ARG0 arg0_val = UnboxScalar::Unbox(arg0); - BoxScalar::Box(Op::template Call(ctx, arg0_val), + Arg0Value arg0_val = UnboxScalar::Unbox(arg0); + BoxScalar::Box(Op::template Call(ctx, arg0_val), out->scalar().get()); } else { out->value = MakeNullScalar(arg0.type); @@ -441,37 +522,13 @@ struct ScalarUnary { } }; -// A VisitArrayDataInline variant that passes a Decimal128 value, -// not util::string_view, for decimal128 arrays, - -template -static typename std::enable_if::value, void>::type -VisitArrayValuesInline(const ArrayData& arr, VisitFunc&& valid_func, - NullFunc&& null_func) { - VisitArrayDataInline(arr, std::forward(valid_func), - std::forward(null_func)); -} - -template -static typename std::enable_if::value, void>::type -VisitArrayValuesInline(const ArrayData& arr, VisitFunc&& valid_func, - NullFunc&& null_func) { - VisitArrayDataInline( - arr, - [&](util::string_view v) { - const auto dec_value = Decimal128(reinterpret_cast(v.data())); - valid_func(dec_value); - }, - std::forward(null_func)); -} - // An alternative to ScalarUnary that Applies a scalar operation with state on // only the not-null values of a single array template struct ScalarUnaryNotNullStateful { using ThisType = ScalarUnaryNotNullStateful; - using OUT = typename GetOutputType::T; - using ARG0 = typename GetViewType::T; + using OutValue = typename GetOutputType::T; + using Arg0Value = typename GetViewType::T; Op op; explicit ScalarUnaryNotNullStateful(Op op) : op(std::move(op)) {} @@ -493,10 +550,12 @@ struct ScalarUnaryNotNullStateful { static void Exec(const ThisType& functor, KernelContext* ctx, const ArrayData& arg0, Datum* out) { ArrayData* out_arr = out->mutable_array(); - auto out_data = out_arr->GetMutableValues(1); + auto out_data = out_arr->GetMutableValues(1); VisitArrayValuesInline( arg0, - [&](ARG0 v) { *out_data++ = functor.op.template Call(ctx, v); }, + [&](Arg0Value v) { + *out_data++ = functor.op.template Call(ctx, v); + }, [&]() { // null ++out_data; @@ -515,7 +574,7 @@ struct ScalarUnaryNotNullStateful { typename TypeTraits::BuilderType builder; VisitArrayValuesInline( arg0, - [&](ARG0 v) { + [&](Arg0Value v) { KERNEL_RETURN_IF_ERROR(ctx, builder.Append(functor.op.Call(ctx, v))); }, [&]() { KERNEL_RETURN_IF_ERROR(ctx, builder.AppendNull()); }); @@ -536,8 +595,8 @@ struct ScalarUnaryNotNullStateful { out_arr->offset, out_arr->length); VisitArrayValuesInline( arg0, - [&](ARG0 v) { - if (functor.op.template Call(ctx, v)) { + [&](Arg0Value v) { + if (functor.op.template Call(ctx, v)) { out_writer.Set(); } out_writer.Next(); @@ -559,8 +618,8 @@ struct ScalarUnaryNotNullStateful { auto out_data = out_arr->GetMutableValues(1); VisitArrayValuesInline( arg0, - [&](ARG0 v) { - functor.op.template Call(ctx, v).ToBytes(out_data); + [&](Arg0Value v) { + functor.op.template Call(ctx, v).ToBytes(out_data); out_data += 16; }, [&]() { out_data += 16; }); @@ -569,8 +628,8 @@ struct ScalarUnaryNotNullStateful { void Scalar(KernelContext* ctx, const Scalar& arg0, Datum* out) { if (arg0.is_valid) { - ARG0 arg0_val = UnboxScalar::Unbox(arg0); - BoxScalar::Box(this->op.template Call(ctx, arg0_val), + Arg0Value arg0_val = UnboxScalar::Unbox(arg0); + BoxScalar::Box(this->op.template Call(ctx, arg0_val), out->scalar().get()); } else { out->value = MakeNullScalar(arg0.type); @@ -591,8 +650,8 @@ struct ScalarUnaryNotNullStateful { // operator requires some initialization use ScalarUnaryNotNullStateful template struct ScalarUnaryNotNull { - using OUT = typename GetOutputType::T; - using ARG0 = typename GetViewType::T; + using OutValue = typename GetOutputType::T; + using Arg0Value = typename GetViewType::T; static void Exec(KernelContext* ctx, const ExecBatch& batch, Datum* out) { // Seed kernel with dummy state @@ -612,39 +671,42 @@ struct ScalarUnaryNotNull { // The "Op" functor should have the form // // struct Op { -// template -// static OUT Call(KernelContext* ctx, ARG0 arg0, ARG1 arg1) { +// template +// static OutValue Call(KernelContext* ctx, Arg0Value arg0, Arg1Value arg1) { // // implementation // } // }; template struct ScalarBinary { - using OUT = typename GetOutputType::T; - using ARG0 = typename GetViewType::T; - using ARG1 = typename GetViewType::T; + using OutValue = typename GetOutputType::T; + using Arg0Value = typename GetViewType::T; + using Arg1Value = typename GetViewType::T; static void ArrayArray(KernelContext* ctx, const ArrayData& arg0, const ArrayData& arg1, Datum* out) { ArrayIterator arg0_it(arg0); ArrayIterator arg1_it(arg1); - OutputAdapter::Write( - ctx, out, [&]() -> OUT { return Op::template Call(ctx, arg0_it(), arg1_it()); }); + OutputAdapter::Write(ctx, out, [&]() -> OutValue { + return Op::template Call(ctx, arg0_it(), arg1_it()); + }); } static void ArrayScalar(KernelContext* ctx, const ArrayData& arg0, const Scalar& arg1, Datum* out) { ArrayIterator arg0_it(arg0); auto arg1_val = UnboxScalar::Unbox(arg1); - OutputAdapter::Write( - ctx, out, [&]() -> OUT { return Op::template Call(ctx, arg0_it(), arg1_val); }); + OutputAdapter::Write(ctx, out, [&]() -> OutValue { + return Op::template Call(ctx, arg0_it(), arg1_val); + }); } static void ScalarArray(KernelContext* ctx, const Scalar& arg0, const ArrayData& arg1, Datum* out) { auto arg0_val = UnboxScalar::Unbox(arg0); ArrayIterator arg1_it(arg1); - OutputAdapter::Write( - ctx, out, [&]() -> OUT { return Op::template Call(ctx, arg0_val, arg1_it()); }); + OutputAdapter::Write(ctx, out, [&]() -> OutValue { + return Op::template Call(ctx, arg0_val, arg1_it()); + }); } static void ScalarScalar(KernelContext* ctx, const Scalar& arg0, const Scalar& arg1, @@ -666,7 +728,6 @@ struct ScalarBinary { } } else { if (batch[1].kind() == Datum::ARRAY) { - // e.g. if we were doing scalar < array, we flip and do array >= scalar return ScalarArray(ctx, *batch[0].scalar(), *batch[1].array(), out); } else { return ScalarScalar(ctx, *batch[0].scalar(), *batch[1].scalar(), out); @@ -675,11 +736,116 @@ struct ScalarBinary { } }; +// An alternative to ScalarBinary that Applies a scalar operation with state on +// only the value pairs which are not-null in both arrays +template +struct ScalarBinaryNotNullStateful { + using ThisType = ScalarBinaryNotNullStateful; + using OutValue = typename GetOutputType::T; + using Arg0Value = typename GetViewType::T; + using Arg1Value = typename GetViewType::T; + + Op op; + explicit ScalarBinaryNotNullStateful(Op op) : op(std::move(op)) {} + + // NOTE: In ArrayExec, Type is really OutputType + + void ArrayArray(KernelContext* ctx, const ArrayData& arg0, const ArrayData& arg1, + Datum* out) { + OutputArrayWriter writer(out->mutable_array()); + VisitTwoArrayValuesInline( + arg0, arg1, + [&](Arg0Value u, Arg1Value v) { + writer.Write(op.template Call(ctx, u, v)); + }, + [&]() { writer.WriteNull(); }); + } + + void ArrayScalar(KernelContext* ctx, const ArrayData& arg0, const Scalar& arg1, + Datum* out) { + OutputArrayWriter writer(out->mutable_array()); + if (arg1.is_valid) { + const auto arg1_val = UnboxScalar::Unbox(arg1); + VisitArrayValuesInline( + arg0, + [&](Arg0Value u) { + writer.Write( + op.template Call(ctx, u, arg1_val)); + }, + [&]() { writer.WriteNull(); }); + } + } + + void ScalarArray(KernelContext* ctx, const Scalar& arg0, const ArrayData& arg1, + Datum* out) { + OutputArrayWriter writer(out->mutable_array()); + if (arg0.is_valid) { + const auto arg0_val = UnboxScalar::Unbox(arg0); + VisitArrayValuesInline( + arg1, + [&](Arg1Value v) { + writer.Write( + op.template Call(ctx, arg0_val, v)); + }, + [&]() { writer.WriteNull(); }); + } + } + + void ScalarScalar(KernelContext* ctx, const Scalar& arg0, const Scalar& arg1, + Datum* out) { + if (arg0.is_valid && arg1.is_valid) { + const auto arg0_val = UnboxScalar::Unbox(arg0); + const auto arg1_val = UnboxScalar::Unbox(arg1); + BoxScalar::Box( + op.template Call(ctx, arg0_val, arg1_val), + out->scalar().get()); + } + } + + void Exec(KernelContext* ctx, const ExecBatch& batch, Datum* out) { + if (batch[0].kind() == Datum::ARRAY) { + if (batch[1].kind() == Datum::ARRAY) { + return ArrayArray(ctx, *batch[0].array(), *batch[1].array(), out); + } else { + return ArrayScalar(ctx, *batch[0].array(), *batch[1].scalar(), out); + } + } else { + if (batch[1].kind() == Datum::ARRAY) { + return ScalarArray(ctx, *batch[0].scalar(), *batch[1].array(), out); + } else { + return ScalarScalar(ctx, *batch[0].scalar(), *batch[1].scalar(), out); + } + } + } +}; + +// An alternative to ScalarBinary that Applies a scalar operation on only +// the value pairs which are not-null in both arrays. +// The operator is not stateful; if the operator requires some initialization +// use ScalarBinaryNotNullStateful. +template +struct ScalarBinaryNotNull { + using OutValue = typename GetOutputType::T; + using Arg0Value = typename GetViewType::T; + using Arg1Value = typename GetViewType::T; + + static void Exec(KernelContext* ctx, const ExecBatch& batch, Datum* out) { + // Seed kernel with dummy state + ScalarBinaryNotNullStateful kernel({}); + return kernel.Exec(ctx, batch, out); + } +}; + // A kernel exec generator for binary kernels where both input types are the // same template using ScalarBinaryEqualTypes = ScalarBinary; +// A kernel exec generator for non-null binary kernels where both input types are the +// same +template +using ScalarBinaryNotNullEqualTypes = ScalarBinaryNotNull; + } // namespace applicator // ---------------------------------------------------------------------- diff --git a/cpp/src/arrow/compute/kernels/scalar_arithmetic.cc b/cpp/src/arrow/compute/kernels/scalar_arithmetic.cc index 1f0cd3785a74..ff6c6fab7c60 100644 --- a/cpp/src/arrow/compute/kernels/scalar_arithmetic.cc +++ b/cpp/src/arrow/compute/kernels/scalar_arithmetic.cc @@ -16,16 +16,22 @@ // under the License. #include "arrow/compute/kernels/common.h" -#include "arrow/util/int_util.h" +#include "arrow/util/int_util_internal.h" #include "arrow/util/macros.h" -#ifndef __has_builtin -#define __has_builtin(x) 0 -#endif - namespace arrow { + +using internal::AddWithOverflow; +using internal::DivideWithOverflow; +using internal::MultiplyWithOverflow; +using internal::SubtractWithOverflow; + namespace compute { namespace internal { + +using applicator::ScalarBinaryEqualTypes; +using applicator::ScalarBinaryNotNullEqualTypes; + namespace { template @@ -72,35 +78,19 @@ struct Add { }; struct AddChecked { -#if __has_builtin(__builtin_add_overflow) - template - static enable_if_integer Call(KernelContext* ctx, T left, T right) { - T result; - if (ARROW_PREDICT_FALSE(__builtin_add_overflow(left, right, &result))) { + template + enable_if_integer Call(KernelContext* ctx, Arg0 left, Arg1 right) { + static_assert(std::is_same::value && std::is_same::value, ""); + T result = 0; + if (ARROW_PREDICT_FALSE(AddWithOverflow(left, right, &result))) { ctx->SetStatus(Status::Invalid("overflow")); } return result; } -#else - template - static enable_if_unsigned_integer Call(KernelContext* ctx, T left, T right) { - if (ARROW_PREDICT_FALSE(arrow::internal::HasPositiveAdditionOverflow(left, right))) { - ctx->SetStatus(Status::Invalid("overflow")); - } - return left + right; - } - template - static enable_if_signed_integer Call(KernelContext* ctx, T left, T right) { - if (ARROW_PREDICT_FALSE(arrow::internal::HasSignedAdditionOverflow(left, right))) { - ctx->SetStatus(Status::Invalid("overflow")); - } - return left + right; - } -#endif - - template - static constexpr enable_if_floating_point Call(KernelContext*, T left, T right) { + template + enable_if_floating_point Call(KernelContext*, Arg0 left, Arg1 right) { + static_assert(std::is_same::value && std::is_same::value, ""); return left + right; } }; @@ -123,36 +113,19 @@ struct Subtract { }; struct SubtractChecked { -#if __has_builtin(__builtin_sub_overflow) - template - static enable_if_integer Call(KernelContext* ctx, T left, T right) { - T result; - if (ARROW_PREDICT_FALSE(__builtin_sub_overflow(left, right, &result))) { + template + enable_if_integer Call(KernelContext* ctx, Arg0 left, Arg1 right) { + static_assert(std::is_same::value && std::is_same::value, ""); + T result = 0; + if (ARROW_PREDICT_FALSE(SubtractWithOverflow(left, right, &result))) { ctx->SetStatus(Status::Invalid("overflow")); } return result; } -#else - template - static enable_if_unsigned_integer Call(KernelContext* ctx, T left, T right) { - if (ARROW_PREDICT_FALSE( - arrow::internal::HasPositiveSubtractionOverflow(left, right))) { - ctx->SetStatus(Status::Invalid("overflow")); - } - return left - right; - } - template - static enable_if_signed_integer Call(KernelContext* ctx, T left, T right) { - if (ARROW_PREDICT_FALSE(arrow::internal::HasSignedSubtractionOverflow(left, right))) { - ctx->SetStatus(Status::Invalid("overflow")); - } - return left - right; - } -#endif - - template - static constexpr enable_if_floating_point Call(KernelContext*, T left, T right) { + template + enable_if_floating_point Call(KernelContext*, Arg0 left, Arg1 right) { + static_assert(std::is_same::value && std::is_same::value, ""); return left - right; } }; @@ -197,59 +170,94 @@ struct Multiply { }; struct MultiplyChecked { - template - static enable_if_integer Call(KernelContext* ctx, T left, T right) { - T result; -#if __has_builtin(__builtin_mul_overflow) - if (ARROW_PREDICT_FALSE(__builtin_mul_overflow(left, right, &result))) { - ctx->SetStatus(Status::Invalid("overflow")); - } -#else - result = Multiply::Call(ctx, left, right); - if (left != 0 && ARROW_PREDICT_FALSE(result / left != right)) { + template + enable_if_integer Call(KernelContext* ctx, Arg0 left, Arg1 right) { + static_assert(std::is_same::value && std::is_same::value, ""); + T result = 0; + if (ARROW_PREDICT_FALSE(MultiplyWithOverflow(left, right, &result))) { ctx->SetStatus(Status::Invalid("overflow")); } -#endif return result; } - template - static constexpr enable_if_floating_point Call(KernelContext*, T left, T right) { + template + enable_if_floating_point Call(KernelContext*, Arg0 left, Arg1 right) { + static_assert(std::is_same::value && std::is_same::value, ""); return left * right; } }; -using applicator::ScalarBinaryEqualTypes; +struct Divide { + template + static enable_if_floating_point Call(KernelContext* ctx, Arg0 left, Arg1 right) { + return left / right; + } + + template + static enable_if_integer Call(KernelContext* ctx, Arg0 left, Arg1 right) { + T result; + if (ARROW_PREDICT_FALSE(DivideWithOverflow(left, right, &result))) { + if (right == 0) { + ctx->SetStatus(Status::Invalid("divide by zero")); + } else { + result = 0; + } + } + return result; + } +}; + +struct DivideChecked { + template + static enable_if_integer Call(KernelContext* ctx, Arg0 left, Arg1 right) { + static_assert(std::is_same::value && std::is_same::value, ""); + T result; + if (ARROW_PREDICT_FALSE(DivideWithOverflow(left, right, &result))) { + if (right == 0) { + ctx->SetStatus(Status::Invalid("divide by zero")); + } else { + ctx->SetStatus(Status::Invalid("overflow")); + } + } + return result; + } + + template + static enable_if_floating_point Call(KernelContext* ctx, Arg0 left, Arg1 right) { + static_assert(std::is_same::value && std::is_same::value, ""); + if (ARROW_PREDICT_FALSE(right == 0)) { + ctx->SetStatus(Status::Invalid("divide by zero")); + return 0; + } + return left / right; + } +}; // Generate a kernel given an arithmetic functor -// -// To avoid undefined behaviour of signed integer overflow treat the signed -// input argument values as unsigned then cast them to signed making them wrap -// around. -template +template