From 0e3e140e98c1b674d258b74aa776192d2ae8510d Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Wed, 5 Aug 2020 11:07:41 +0900 Subject: [PATCH 001/460] ARROW-9650: [Packaging][APT] Drop support for Ubuntu 19.10 Because it reached EOL. Closes #7902 from kou/apt-remove-ubuntu-19.10 Authored-by: Sutou Kouhei Signed-off-by: Sutou Kouhei --- dev/release/binary-task.rb | 1 - dev/release/verify-release-candidate.sh | 2 - .../apt/ubuntu-eoan/Dockerfile | 41 -------- .../apache-arrow/apt/ubuntu-eoan-arm64/from | 18 ---- .../apache-arrow/apt/ubuntu-eoan/Dockerfile | 89 ----------------- .../apt/ubuntu-eoan/qemu-dummy-static | 33 ------- dev/tasks/linux-packages/package-task.rb | 2 - dev/tasks/tasks.yml | 99 ------------------- 8 files changed, 285 deletions(-) delete mode 100644 dev/tasks/linux-packages/apache-arrow-archive-keyring/apt/ubuntu-eoan/Dockerfile delete mode 100644 dev/tasks/linux-packages/apache-arrow/apt/ubuntu-eoan-arm64/from delete mode 100644 dev/tasks/linux-packages/apache-arrow/apt/ubuntu-eoan/Dockerfile delete mode 100755 dev/tasks/linux-packages/apache-arrow/apt/ubuntu-eoan/qemu-dummy-static diff --git a/dev/release/binary-task.rb b/dev/release/binary-task.rb index 2e677c416be1..89d601d8b9bc 100644 --- a/dev/release/binary-task.rb +++ b/dev/release/binary-task.rb @@ -1025,7 +1025,6 @@ def available_apt_targets ["debian", "buster", "main"], ["ubuntu", "xenial", "main"], ["ubuntu", "bionic", "main"], - ["ubuntu", "eoan", "main"], ["ubuntu", "focal", "main"], ] end diff --git a/dev/release/verify-release-candidate.sh b/dev/release/verify-release-candidate.sh index e6a0284107ca..e44692e59cbe 100755 --- a/dev/release/verify-release-candidate.sh +++ b/dev/release/verify-release-candidate.sh @@ -143,8 +143,6 @@ test_apt() { "arm64v8/ubuntu:xenial" \ "ubuntu:bionic" \ "arm64v8/ubuntu:bionic" \ - "ubuntu:eoan" \ - "arm64v8/ubuntu:eoan" \ "ubuntu:focal" \ "arm64v8/ubuntu:focal"; do \ case "${target}" in diff --git a/dev/tasks/linux-packages/apache-arrow-archive-keyring/apt/ubuntu-eoan/Dockerfile b/dev/tasks/linux-packages/apache-arrow-archive-keyring/apt/ubuntu-eoan/Dockerfile deleted file mode 100644 index 884710d61a0f..000000000000 --- a/dev/tasks/linux-packages/apache-arrow-archive-keyring/apt/ubuntu-eoan/Dockerfile +++ /dev/null @@ -1,41 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -FROM ubuntu:eoan - -RUN \ - echo "debconf debconf/frontend select Noninteractive" | \ - debconf-set-selections - -RUN \ - echo 'APT::Install-Recommends "false";' > \ - /etc/apt/apt.conf.d/disable-install-recommends - -ARG DEBUG - -RUN \ - quiet=$([ "${DEBUG}" = "yes" ] || echo "-qq") && \ - apt update ${quiet} && \ - apt install -y -V ${quiet} \ - build-essential \ - debhelper \ - devscripts \ - fakeroot \ - gnupg \ - lsb-release && \ - apt clean && \ - rm -rf /var/lib/apt/lists/* diff --git a/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-eoan-arm64/from b/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-eoan-arm64/from deleted file mode 100644 index 7c4458fc9776..000000000000 --- a/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-eoan-arm64/from +++ /dev/null @@ -1,18 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -arm64v8/ubuntu:eoan diff --git a/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-eoan/Dockerfile b/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-eoan/Dockerfile deleted file mode 100644 index dceb8ba3a08c..000000000000 --- a/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-eoan/Dockerfile +++ /dev/null @@ -1,89 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -ARG FROM=ubuntu:eoan -FROM ${FROM} - -COPY qemu-* /usr/bin/ - -RUN \ - echo "debconf debconf/frontend select Noninteractive" | \ - debconf-set-selections - -RUN \ - echo 'APT::Install-Recommends "false";' > \ - /etc/apt/apt.conf.d/disable-install-recommends - -ARG DEBUG -ARG LLVM -RUN \ - quiet=$([ "${DEBUG}" = "yes" ] || echo "-qq") && \ - apt update ${quiet} && \ - apt install -y -V ${quiet} \ - apt-transport-https \ - ca-certificates \ - gnupg \ - wget && \ - wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - && \ - echo "deb https://apt.llvm.org/eoan/ llvm-toolchain-eoan-${LLVM} main" > \ - /etc/apt/sources.list.d/llvm.list && \ - apt update ${quiet} && \ - apt install -y -V ${quiet} \ - build-essential \ - cmake \ - debhelper \ - devscripts \ - git \ - gtk-doc-tools \ - libboost-filesystem-dev \ - libboost-regex-dev \ - libboost-system-dev \ - libbrotli-dev \ - libbz2-dev \ - libgirepository1.0-dev \ - libglib2.0-doc \ - libgmock-dev \ - libgoogle-glog-dev \ - libgtest-dev \ - liblz4-dev \ - libre2-dev \ - libsnappy-dev \ - libssl-dev \ - libthrift-dev \ - libutf8proc-dev \ - libzstd-dev \ - lsb-release \ - ninja-build \ - pkg-config \ - python3-dev \ - python3-numpy \ - python3-pip \ - rapidjson-dev \ - tzdata \ - zlib1g-dev && \ - if [ "$(dpkg --print-architecture)" != "arm64" ]; then \ - apt install -y -V ${quiet} \ - clang-${LLVM} \ - llvm-${LLVM}-dev; \ - fi && \ - if apt list | grep '^nvidia-cuda-toolkit/'; then \ - apt install -y -V ${quiet} nvidia-cuda-toolkit; \ - fi && \ - apt clean && \ - pip3 install --upgrade meson && \ - ln -s /usr/local/bin/meson /usr/bin/ && \ - rm -rf /var/lib/apt/lists/* diff --git a/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-eoan/qemu-dummy-static b/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-eoan/qemu-dummy-static deleted file mode 100755 index c42e0962def3..000000000000 --- a/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-eoan/qemu-dummy-static +++ /dev/null @@ -1,33 +0,0 @@ -#!/bin/sh -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# Do nothing. This exists only for not requiring qemu-aarch64-static copy. -# Recent Debian (buster or later) and Ubuntu (18.10 or later) on amd64 hosts or -# arm64 host don't require qemu-aarch64-static in Docker image. But old Debian -# and Ubuntu hosts on amd64 require qemu-aarch64-static in Docker image. -# -# We use "COPY qemu* /usr/bin/" in Dockerfile. If we don't put any "qemnu*", -# the "COPY" is failed. It means that we always require "qemu*" even if we -# use recent Debian/Ubuntu or arm64 host. If we have this dummy "qemu*" file, -# the "COPY" isn't failed. It means that we can copy "qemu*" only when we -# need. -# -# See also "script" in dev/tasks/linux-packages/azure.linux.arm64.yml. -# Azure Pipelines uses old Ubuntu (18.04). -# So we need to put "qemu-aarch64-static" into this directory. diff --git a/dev/tasks/linux-packages/package-task.rb b/dev/tasks/linux-packages/package-task.rb index 2ea6845ed28d..1762e9fd39bb 100644 --- a/dev/tasks/linux-packages/package-task.rb +++ b/dev/tasks/linux-packages/package-task.rb @@ -239,8 +239,6 @@ def apt_targets_default # "ubuntu-xenial-arm64", "ubuntu-bionic", # "ubuntu-bionic-arm64", - "ubuntu-eoan", - # "ubuntu-eoan-arm64", "ubuntu-focal", # "ubuntu-focal-arm64", ] diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml index a1dca607797b..3fc9ab17f393 100644 --- a/dev/tasks/tasks.yml +++ b/dev/tasks/tasks.yml @@ -921,105 +921,6 @@ tasks: - libparquet-glib200_{no_rc_version}-1_[a-z0-9]+.deb - libparquet200_{no_rc_version}-1_[a-z0-9]+.deb - ubuntu-eoan-amd64: - ci: github - template: linux-packages/github.linux.amd64.yml - params: - build_task: "apt:build" - target: "ubuntu-eoan" - upload_extensions: - - .ddeb - - .deb - - .debian.tar.xz - - .dsc - - .orig.tar.gz - artifacts: - - apache-arrow-archive-keyring_{no_rc_version}-1.debian.tar.xz - - apache-arrow-archive-keyring_{no_rc_version}-1.dsc - - apache-arrow-archive-keyring_{no_rc_version}-1_all.deb - - apache-arrow-archive-keyring_{no_rc_version}.orig.tar.gz - - apache-arrow_{no_rc_version}-1.debian.tar.xz - - apache-arrow_{no_rc_version}-1.dsc - - apache-arrow_{no_rc_version}.orig.tar.gz - - gir1.2-arrow-1.0_{no_rc_version}-1_[a-z0-9]+.deb - - gir1.2-arrow-cuda-1.0_{no_rc_version}-1_[a-z0-9]+.deb - - gir1.2-gandiva-1.0_{no_rc_version}-1_[a-z0-9]+.deb - - gir1.2-parquet-1.0_{no_rc_version}-1_[a-z0-9]+.deb - - gir1.2-plasma-1.0_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow-dev_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow-glib-dev_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow-glib-doc_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow-glib200_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow-cuda-dev_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow-cuda-glib-dev_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow-cuda-glib200_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow-cuda200_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow-dataset-dev_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow-dataset-glib-dev_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow-dataset-glib-doc_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow-dataset-glib200_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow-dataset200_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow-flight-dev_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow-flight200_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow-python-dev_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow-python200_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow200_{no_rc_version}-1_[a-z0-9]+.deb - - libgandiva-dev_{no_rc_version}-1_[a-z0-9]+.deb - - libgandiva-glib-dev_{no_rc_version}-1_[a-z0-9]+.deb - - libgandiva-glib-doc_{no_rc_version}-1_[a-z0-9]+.deb - - libgandiva-glib200_{no_rc_version}-1_[a-z0-9]+.deb - - libgandiva200_{no_rc_version}-1_[a-z0-9]+.deb - - libparquet-dev_{no_rc_version}-1_[a-z0-9]+.deb - - libparquet-glib-dev_{no_rc_version}-1_[a-z0-9]+.deb - - libparquet-glib-doc_{no_rc_version}-1_[a-z0-9]+.deb - - libparquet-glib200_{no_rc_version}-1_[a-z0-9]+.deb - - libparquet200_{no_rc_version}-1_[a-z0-9]+.deb - - libplasma-dev_{no_rc_version}-1_[a-z0-9]+.deb - - libplasma-glib-dev_{no_rc_version}-1_[a-z0-9]+.deb - - libplasma-glib-doc_{no_rc_version}-1_[a-z0-9]+.deb - - libplasma-glib200_{no_rc_version}-1_[a-z0-9]+.deb - - libplasma200_{no_rc_version}-1_[a-z0-9]+.deb - - plasma-store-server-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb - - plasma-store-server_{no_rc_version}-1_[a-z0-9]+.deb - - ubuntu-eoan-arm64: - ci: travis - template: linux-packages/travis.linux.arm64.yml - params: - build_task: "apt:build" - target: "ubuntu-eoan-arm64" - upload_extensions: - - .ddeb - - .deb - - .debian.tar.xz - - .dsc - - .orig.tar.gz - artifacts: - - apache-arrow_{no_rc_version}-1.debian.tar.xz - - apache-arrow_{no_rc_version}-1.dsc - - apache-arrow_{no_rc_version}.orig.tar.gz - - gir1.2-arrow-1.0_{no_rc_version}-1_[a-z0-9]+.deb - - gir1.2-parquet-1.0_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow-dev_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow-glib-dev_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow-glib-doc_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow-glib200_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow-dataset-dev_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow-dataset-glib-dev_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow-dataset-glib-doc_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow-dataset-glib200_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow-dataset200_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow-flight-dev_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow-flight200_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow-python-dev_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow-python200_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow200_{no_rc_version}-1_[a-z0-9]+.deb - - libparquet-dev_{no_rc_version}-1_[a-z0-9]+.deb - - libparquet-glib-dev_{no_rc_version}-1_[a-z0-9]+.deb - - libparquet-glib-doc_{no_rc_version}-1_[a-z0-9]+.deb - - libparquet-glib200_{no_rc_version}-1_[a-z0-9]+.deb - - libparquet200_{no_rc_version}-1_[a-z0-9]+.deb - ubuntu-focal-amd64: ci: github template: linux-packages/github.linux.amd64.yml From ef2ccfc7620add5f365b4391dcf61d5c01da65a9 Mon Sep 17 00:00:00 2001 From: Benjamin Kietzman Date: Wed, 5 Aug 2020 14:14:09 -0400 Subject: [PATCH 002/460] ARROW-8002: [C++][Dataset][R] Support partitioned dataset writing Closes #7869 from bkietz/8002-partitioned-write Lead-authored-by: Benjamin Kietzman Co-authored-by: Neal Richardson Signed-off-by: Benjamin Kietzman --- c_glib/arrow-glib/compute.cpp | 3 +- cpp/src/arrow/array/array_nested.cc | 17 +- cpp/src/arrow/array/array_nested.h | 10 +- cpp/src/arrow/array/array_view_test.cc | 11 + cpp/src/arrow/compute/api_vector.cc | 9 +- cpp/src/arrow/compute/api_vector.h | 4 +- .../arrow/compute/kernels/vector_hash_test.cc | 15 +- cpp/src/arrow/dataset/dataset_internal.h | 4 +- cpp/src/arrow/dataset/dataset_test.cc | 2 +- cpp/src/arrow/dataset/file_base.cc | 140 ++++--- cpp/src/arrow/dataset/file_base.h | 132 +------ cpp/src/arrow/dataset/file_ipc.cc | 55 +-- cpp/src/arrow/dataset/file_ipc.h | 6 +- cpp/src/arrow/dataset/file_ipc_test.cc | 353 +++++++++++++++--- cpp/src/arrow/dataset/filter.cc | 188 ++++++++++ cpp/src/arrow/dataset/filter.h | 17 + cpp/src/arrow/dataset/filter_test.cc | 52 +++ cpp/src/arrow/dataset/partition.cc | 321 ++++------------ cpp/src/arrow/dataset/partition.h | 27 +- cpp/src/arrow/dataset/partition_test.cc | 181 +-------- cpp/src/arrow/dataset/scanner_internal.h | 37 ++ cpp/src/arrow/dataset/test_util.h | 8 +- cpp/src/arrow/dataset/type_fwd.h | 3 - cpp/src/arrow/record_batch.cc | 2 +- cpp/src/arrow/record_batch.h | 7 +- r/DESCRIPTION | 1 + r/NAMESPACE | 2 + r/R/arrowExports.R | 4 + r/R/dataset-write.R | 82 ++++ r/R/dataset.R | 18 +- r/R/schema.R | 24 ++ r/man/Dataset.Rd | 3 + r/man/write_dataset.Rd | 42 +++ r/src/arrowExports.cpp | 22 ++ r/src/dataset.cpp | 15 + r/tests/testthat/test-dataset.R | 101 +++++ r/tests/testthat/test-schema.R | 18 + 37 files changed, 1193 insertions(+), 743 deletions(-) create mode 100644 r/R/dataset-write.R create mode 100644 r/man/write_dataset.Rd diff --git a/c_glib/arrow-glib/compute.cpp b/c_glib/arrow-glib/compute.cpp index 3e318999d31d..20d910e3250e 100644 --- a/c_glib/arrow-glib/compute.cpp +++ b/c_glib/arrow-glib/compute.cpp @@ -1246,7 +1246,8 @@ garrow_array_count_values(GArrowArray *array, auto arrow_array = garrow_array_get_raw(array); auto arrow_counted_values = arrow::compute::ValueCounts(arrow_array); if (garrow::check(error, arrow_counted_values, "[array][count-values]")) { - return GARROW_STRUCT_ARRAY(garrow_array_new_raw(&(*arrow_counted_values))); + std::shared_ptr arrow_counted_values_array = *arrow_counted_values; + return GARROW_STRUCT_ARRAY(garrow_array_new_raw(&arrow_counted_values_array)); } else { return NULL; } diff --git a/cpp/src/arrow/array/array_nested.cc b/cpp/src/arrow/array/array_nested.cc index e8d4ed92443d..f1e8d329393d 100644 --- a/cpp/src/arrow/array/array_nested.cc +++ b/cpp/src/arrow/array/array_nested.cc @@ -101,9 +101,8 @@ Status CleanListOffsets(const Array& offsets, MemoryPool* pool, } template -Result> ListArrayFromArrays(const Array& offsets, - const Array& values, - MemoryPool* pool) { +Result::ArrayType>> ListArrayFromArrays( + const Array& offsets, const Array& values, MemoryPool* pool) { using offset_type = typename TYPE::offset_type; using ArrayType = typename TypeTraits::ArrayType; using OffsetArrowType = typename CTypeTraits::ArrowType; @@ -238,15 +237,15 @@ void LargeListArray::SetData(const std::shared_ptr& data) { values_ = MakeArray(data_->child_data[0]); } -Result> ListArray::FromArrays(const Array& offsets, - const Array& values, - MemoryPool* pool) { +Result> ListArray::FromArrays(const Array& offsets, + const Array& values, + MemoryPool* pool) { return ListArrayFromArrays(offsets, values, pool); } -Result> LargeListArray::FromArrays(const Array& offsets, - const Array& values, - MemoryPool* pool) { +Result> LargeListArray::FromArrays(const Array& offsets, + const Array& values, + MemoryPool* pool) { return ListArrayFromArrays(offsets, values, pool); } diff --git a/cpp/src/arrow/array/array_nested.h b/cpp/src/arrow/array/array_nested.h index e37c34babc10..e5a219c98d28 100644 --- a/cpp/src/arrow/array/array_nested.h +++ b/cpp/src/arrow/array/array_nested.h @@ -103,7 +103,7 @@ class ARROW_EXPORT ListArray : public BaseListArray { /// \param[in] values Array containing list values /// \param[in] pool MemoryPool in case new offsets array needs to be /// allocated because of null values - static Result> FromArrays( + static Result> FromArrays( const Array& offsets, const Array& values, MemoryPool* pool = default_memory_pool()); @@ -148,7 +148,7 @@ class ARROW_EXPORT LargeListArray : public BaseListArray { /// \param[in] values Array containing list values /// \param[in] pool MemoryPool in case new offsets array needs to be /// allocated because of null values - static Result> FromArrays( + static Result> FromArrays( const Array& offsets, const Array& values, MemoryPool* pool = default_memory_pool()); @@ -310,8 +310,7 @@ class ARROW_EXPORT StructArray : public Array { /// The length and data type are automatically inferred from the arguments. /// There should be at least one child array. static Result> Make( - const std::vector>& children, - const std::vector& field_names, + const ArrayVector& children, const std::vector& field_names, std::shared_ptr null_bitmap = NULLPTR, int64_t null_count = kUnknownNullCount, int64_t offset = 0); @@ -321,8 +320,7 @@ class ARROW_EXPORT StructArray : public Array { /// There should be at least one child array. This method does not /// check that field types and child array types are consistent. static Result> Make( - const std::vector>& children, - const std::vector>& fields, + const ArrayVector& children, const FieldVector& fields, std::shared_ptr null_bitmap = NULLPTR, int64_t null_count = kUnknownNullCount, int64_t offset = 0); diff --git a/cpp/src/arrow/array/array_view_test.cc b/cpp/src/arrow/array/array_view_test.cc index 3aac62da0dee..e73bbda7abc5 100644 --- a/cpp/src/arrow/array/array_view_test.cc +++ b/cpp/src/arrow/array/array_view_test.cc @@ -329,6 +329,17 @@ TEST(TestArrayView, FixedSizeListAsFlat) { // XXX With nulls (currently fails) } +TEST(TestArrayView, FixedSizeListAsFixedSizeBinary) { + auto ty1 = fixed_size_list(int32(), 1); +#if ARROW_LITTLE_ENDIAN + auto arr = ArrayFromJSON(ty1, "[[2020568934], [2054316386]]"); +#else + auto arr = ArrayFromJSON(ty1, "[[1718579064], [1650553466]]"); +#endif + auto expected = ArrayFromJSON(fixed_size_binary(4), R"(["foox", "barz"])"); + CheckView(arr, expected); +} + TEST(TestArrayView, SparseUnionAsStruct) { auto child1 = ArrayFromJSON(int16(), "[0, -1, 42]"); auto child2 = ArrayFromJSON(int32(), "[0, 1069547520, -1071644672]"); diff --git a/cpp/src/arrow/compute/api_vector.cc b/cpp/src/arrow/compute/api_vector.cc index 9a36714d107f..1f6972860aea 100644 --- a/cpp/src/arrow/compute/api_vector.cc +++ b/cpp/src/arrow/compute/api_vector.cc @@ -21,13 +21,18 @@ #include #include +#include "arrow/array/array_nested.h" #include "arrow/array/builder_primitive.h" #include "arrow/compute/exec.h" #include "arrow/datum.h" #include "arrow/record_batch.h" #include "arrow/result.h" +#include "arrow/util/checked_cast.h" namespace arrow { + +using internal::checked_pointer_cast; + namespace compute { // ---------------------------------------------------------------------- @@ -60,9 +65,9 @@ const char kCountsFieldName[] = "counts"; const int32_t kValuesFieldIndex = 0; const int32_t kCountsFieldIndex = 1; -Result> ValueCounts(const Datum& value, ExecContext* ctx) { +Result> ValueCounts(const Datum& value, ExecContext* ctx) { ARROW_ASSIGN_OR_RAISE(Datum result, CallFunction("value_counts", {value}, ctx)); - return result.make_array(); + return checked_pointer_cast(result.make_array()); } // ---------------------------------------------------------------------- diff --git a/cpp/src/arrow/compute/api_vector.h b/cpp/src/arrow/compute/api_vector.h index 3aa3434c0982..de36202f0196 100644 --- a/cpp/src/arrow/compute/api_vector.h +++ b/cpp/src/arrow/compute/api_vector.h @@ -202,8 +202,8 @@ ARROW_EXPORT extern const int32_t kCountsFieldIndex; /// \since 1.0.0 /// \note API not yet finalized ARROW_EXPORT -Result> ValueCounts(const Datum& value, - ExecContext* ctx = NULLPTR); +Result> ValueCounts(const Datum& value, + ExecContext* ctx = NULLPTR); /// \brief Dictionary-encode values in an array-like object /// \param[in] data array-like input diff --git a/cpp/src/arrow/compute/kernels/vector_hash_test.cc b/cpp/src/arrow/compute/kernels/vector_hash_test.cc index 70ed84b09adb..10562e969510 100644 --- a/cpp/src/arrow/compute/kernels/vector_hash_test.cc +++ b/cpp/src/arrow/compute/kernels/vector_hash_test.cc @@ -82,9 +82,8 @@ void CheckValueCountsNull(const std::shared_ptr& type) { std::shared_ptr ex_values = ArrayFromJSON(type, "[]"); std::shared_ptr ex_counts = ArrayFromJSON(int64(), "[]"); - ASSERT_OK_AND_ASSIGN(std::shared_ptr result, ValueCounts(input)); - ASSERT_OK(result->ValidateFull()); - auto result_struct = std::dynamic_pointer_cast(result); + ASSERT_OK_AND_ASSIGN(auto result_struct, ValueCounts(input)); + ASSERT_OK(result_struct->ValidateFull()); ASSERT_NE(result_struct->GetFieldByName(kValuesFieldName), nullptr); // TODO: We probably shouldn't rely on value ordering. ASSERT_ARRAYS_EQUAL(*ex_values, *result_struct->GetFieldByName(kValuesFieldName)); @@ -615,8 +614,7 @@ TEST_F(TestHashKernel, ChunkedArrayInvoke) { std::vector dict_values = {"foo", "bar", "baz", "quuux"}; auto ex_dict = _MakeArray(type, dict_values, {}); - std::vector counts = {3, 2, 1, 1}; - auto ex_counts = _MakeArray(int64(), counts, {}); + auto ex_counts = _MakeArray(int64(), {3, 2, 1, 1}, {}); ArrayVector arrays = {a1, a2}; auto carr = std::make_shared(arrays); @@ -636,10 +634,9 @@ TEST_F(TestHashKernel, ChunkedArrayInvoke) { auto dict_carr = std::make_shared(dict_arrays); // Unique counts - ASSERT_OK_AND_ASSIGN(std::shared_ptr counts_array, ValueCounts(carr)); - auto counts_struct = std::dynamic_pointer_cast(counts_array); - ASSERT_ARRAYS_EQUAL(*ex_dict, *counts_struct->field(0)); - ASSERT_ARRAYS_EQUAL(*ex_counts, *counts_struct->field(1)); + ASSERT_OK_AND_ASSIGN(auto counts, ValueCounts(carr)); + ASSERT_ARRAYS_EQUAL(*ex_dict, *counts->field(0)); + ASSERT_ARRAYS_EQUAL(*ex_counts, *counts->field(1)); // Dictionary encode ASSERT_OK_AND_ASSIGN(Datum encoded_out, DictionaryEncode(carr)); diff --git a/cpp/src/arrow/dataset/dataset_internal.h b/cpp/src/arrow/dataset/dataset_internal.h index 40ffab513c20..489339e79078 100644 --- a/cpp/src/arrow/dataset/dataset_internal.h +++ b/cpp/src/arrow/dataset/dataset_internal.h @@ -23,6 +23,7 @@ #include #include "arrow/dataset/dataset.h" +#include "arrow/dataset/file_base.h" #include "arrow/dataset/type_fwd.h" #include "arrow/record_batch.h" #include "arrow/scalar.h" @@ -51,7 +52,8 @@ inline FragmentIterator GetFragmentsFromDatasets(const DatasetVector& datasets, return MakeFlattenIterator(std::move(fragments_it)); } -inline RecordBatchIterator IteratorFromReader(std::shared_ptr reader) { +inline RecordBatchIterator IteratorFromReader( + const std::shared_ptr& reader) { return MakeFunctionIterator([reader] { return reader->Next(); }); } diff --git a/cpp/src/arrow/dataset/dataset_test.cc b/cpp/src/arrow/dataset/dataset_test.cc index e9430aa87e39..7a378cd92107 100644 --- a/cpp/src/arrow/dataset/dataset_test.cc +++ b/cpp/src/arrow/dataset/dataset_test.cc @@ -385,7 +385,7 @@ class TestEndToEnd : public TestUnionDataset { auto mock_fs = std::make_shared(fs::kNoTime); for (const auto& f : files) { - ARROW_EXPECT_OK(mock_fs->CreateFile(f.first, f.second, /* recursive */ true)); + ARROW_EXPECT_OK(mock_fs->CreateFile(f.first, f.second, /*recursive=*/true)); } fs_ = mock_fs; diff --git a/cpp/src/arrow/dataset/file_base.cc b/cpp/src/arrow/dataset/file_base.cc index 806c625d52b1..caa44c6edf7e 100644 --- a/cpp/src/arrow/dataset/file_base.cc +++ b/cpp/src/arrow/dataset/file_base.cc @@ -23,37 +23,31 @@ #include "arrow/dataset/dataset_internal.h" #include "arrow/dataset/filter.h" #include "arrow/dataset/scanner.h" +#include "arrow/dataset/scanner_internal.h" #include "arrow/filesystem/filesystem.h" #include "arrow/filesystem/localfs.h" #include "arrow/filesystem/path_util.h" #include "arrow/io/interfaces.h" #include "arrow/io/memory.h" #include "arrow/util/iterator.h" +#include "arrow/util/logging.h" #include "arrow/util/task_group.h" namespace arrow { namespace dataset { -Result> FileSource::Open() const { +Result> FileSource::Open() const { if (filesystem_) { return filesystem_->OpenInputFile(file_info_); } if (buffer_) { - return std::make_shared<::arrow::io::BufferReader>(buffer_); + return std::make_shared(buffer_); } return custom_open_(); } -Result> WritableFileSource::Open() const { - if (filesystem_) { - return filesystem_->OpenOutputStream(path_); - } - - return std::make_shared<::arrow::io::BufferOutputStream>(buffer_); -} - Result> FileFormat::MakeFragment( FileSource source, std::shared_ptr physical_schema) { return MakeFragment(std::move(source), scalar(true), std::move(physical_schema)); @@ -71,11 +65,8 @@ Result> FileFormat::MakeFragment( new FileFragment(std::move(source), shared_from_this(), std::move(partition_expression), std::move(physical_schema))); } - -Result> FileFormat::WriteFragment( - WritableFileSource destination, std::shared_ptr fragment, - std::shared_ptr scan_options, - std::shared_ptr scan_context) { +Status FileFormat::WriteFragment(RecordBatchReader* batches, + io::OutputStream* destination) { return Status::NotImplemented("writing fragment of format ", type_name()); } @@ -154,52 +145,99 @@ FragmentIterator FileSystemDataset::GetFragmentsImpl( return MakeVectorIterator(std::move(fragments)); } -Result> FileSystemDataset::Write( - const WritePlan& plan, std::shared_ptr scan_options, - std::shared_ptr scan_context) { - auto filesystem = plan.filesystem; - if (filesystem == nullptr) { - filesystem = std::make_shared(); - } +struct WriteTask { + Status Execute(); - auto task_group = scan_context->TaskGroup(); - auto partition_base_dir = fs::internal::EnsureTrailingSlash(plan.partition_base_dir); - auto extension = "." + plan.format->type_name(); - - std::vector> fragments; - for (size_t i = 0; i < plan.paths.size(); ++i) { - const auto& op = plan.fragment_or_partition_expressions[i]; - if (op.kind() == WritePlan::FragmentOrPartitionExpression::FRAGMENT) { - auto path = partition_base_dir + plan.paths[i] + extension; - - const auto& input_fragment = op.fragment(); - FileSource dest(path, filesystem); - - ARROW_ASSIGN_OR_RAISE(auto write_task, - plan.format->WriteFragment({path, filesystem}, input_fragment, - scan_options, scan_context)); - task_group->Append([write_task] { return write_task->Execute(); }); - - ARROW_ASSIGN_OR_RAISE( - auto fragment, plan.format->MakeFragment( - {path, filesystem}, input_fragment->partition_expression())); - fragments.push_back(std::move(fragment)); + /// The basename of files written by this WriteTask. Extensions + /// are derived from format + std::string basename; + + /// The partitioning with which paths will be generated + std::shared_ptr partitioning; + + /// The format in which fragments will be written + std::shared_ptr format; + + /// The FileSystem and base directory into which fragments will be written + std::shared_ptr filesystem; + std::string base_dir; + + /// Batches to be written + std::shared_ptr batches; + + /// An Expression already satisfied by every batch to be written + std::shared_ptr partition_expression; +}; + +Status WriteTask::Execute() { + std::unordered_map path_to_batches; + + // TODO(bkietz) these calls to Partition() should be scattered across a TaskGroup + for (auto maybe_batch : IteratorFromReader(batches)) { + ARROW_ASSIGN_OR_RAISE(auto batch, std::move(maybe_batch)); + ARROW_ASSIGN_OR_RAISE(auto partitioned_batches, partitioning->Partition(batch)); + for (auto&& partitioned_batch : partitioned_batches) { + AndExpression expr(std::move(partitioned_batch.partition_expression), + partition_expression); + ARROW_ASSIGN_OR_RAISE(std::string path, partitioning->Format(expr)); + path = fs::internal::EnsureLeadingSlash(path); + path_to_batches[path].push_back(std::move(partitioned_batch.batch)); } } - RETURN_NOT_OK(task_group->Finish()); + for (auto&& path_batches : path_to_batches) { + auto dir = base_dir + path_batches.first; + RETURN_NOT_OK(filesystem->CreateDir(dir, /*recursive=*/true)); - return Make(plan.schema, scalar(true), plan.format, fragments); -} + auto path = fs::internal::ConcatAbstractPath(dir, basename); + ARROW_ASSIGN_OR_RAISE(auto destination, filesystem->OpenOutputStream(path)); -Status WriteTask::CreateDestinationParentDir() const { - if (auto filesystem = destination_.filesystem()) { - auto parent = fs::internal::GetAbstractPathParent(destination_.path()).first; - return filesystem->CreateDir(parent, /* recursive = */ true); + DCHECK(!path_batches.second.empty()); + ARROW_ASSIGN_OR_RAISE(auto reader, + RecordBatchReader::Make(std::move(path_batches.second))); + RETURN_NOT_OK(format->WriteFragment(reader.get(), destination.get())); } return Status::OK(); } +Status FileSystemDataset::Write(std::shared_ptr schema, + std::shared_ptr format, + std::shared_ptr filesystem, + std::string base_dir, + std::shared_ptr partitioning, + std::shared_ptr scan_context, + FragmentIterator fragment_it) { + auto task_group = scan_context->TaskGroup(); + + base_dir = fs::internal::RemoveTrailingSlash(base_dir).to_string(); + + for (const auto& f : partitioning->schema()->fields()) { + if (f->type()->id() == Type::DICTIONARY) { + return Status::NotImplemented("writing with dictionary partitions"); + } + } + + int i = 0; + for (auto maybe_fragment : fragment_it) { + ARROW_ASSIGN_OR_RAISE(auto fragment, std::move(maybe_fragment)); + auto task = std::make_shared(); + + task->basename = "dat_" + std::to_string(i++) + "." + format->type_name(); + task->partition_expression = fragment->partition_expression(); + task->format = format; + task->filesystem = filesystem; + task->base_dir = base_dir; + task->partitioning = partitioning; + + // make a record batch reader which yields from a fragment + ARROW_ASSIGN_OR_RAISE(task->batches, FragmentRecordBatchReader::Make( + std::move(fragment), schema, scan_context)); + task_group->Append([task] { return task->Execute(); }); + } + + return task_group->Finish(); +} + } // namespace dataset } // namespace arrow diff --git a/cpp/src/arrow/dataset/file_base.h b/cpp/src/arrow/dataset/file_base.h index 246e71dd6d54..c64714a364c2 100644 --- a/cpp/src/arrow/dataset/file_base.h +++ b/cpp/src/arrow/dataset/file_base.h @@ -117,45 +117,6 @@ class ARROW_DS_EXPORT FileSource { Compression::type compression_ = Compression::UNCOMPRESSED; }; -/// \brief The path and filesystem where an actual file is located or a buffer which can -/// be written to like a file -class ARROW_DS_EXPORT WritableFileSource { - public: - WritableFileSource(std::string path, std::shared_ptr filesystem, - Compression::type compression = Compression::UNCOMPRESSED) - : path_(std::move(path)), - filesystem_(std::move(filesystem)), - compression_(compression) {} - - explicit WritableFileSource(std::shared_ptr buffer, - Compression::type compression = Compression::UNCOMPRESSED) - : buffer_(std::move(buffer)), compression_(compression) {} - - /// \brief Return the type of raw compression on the file, if any - Compression::type compression() const { return compression_; } - - /// \brief Return the file path, if any. Only valid when file source wraps a path. - const std::string& path() const { - static std::string buffer_path = ""; - return filesystem_ ? path_ : buffer_path; - } - - /// \brief Return the filesystem, if any. Otherwise returns nullptr - const std::shared_ptr& filesystem() const { return filesystem_; } - - /// \brief Return the buffer containing the file, if any. Otherwise returns nullptr - const std::shared_ptr& buffer() const { return buffer_; } - - /// \brief Get an OutputStream which wraps this file source - Result> Open() const; - - private: - std::string path_; - std::shared_ptr filesystem_; - std::shared_ptr buffer_; - Compression::type compression_ = Compression::UNCOMPRESSED; -}; - /// \brief Base class for file format implementation class ARROW_DS_EXPORT FileFormat : public std::enable_shared_from_this { public: @@ -190,12 +151,9 @@ class ARROW_DS_EXPORT FileFormat : public std::enable_shared_from_this> MakeFragment( FileSource source, std::shared_ptr physical_schema = NULLPTR); - /// \brief Write a fragment. If the parent directory of destination does not exist, it - /// will be created. - virtual Result> WriteFragment( - WritableFileSource destination, std::shared_ptr fragment, - std::shared_ptr options, - std::shared_ptr scan_context); // FIXME(bkietz) make this pure virtual + /// \brief Write a fragment. + /// FIXME(bkietz) make this pure virtual + virtual Status WriteFragment(RecordBatchReader* batches, io::OutputStream* destination); }; /// \brief A Fragment that is stored in a file with a known format @@ -248,14 +206,20 @@ class ARROW_DS_EXPORT FileSystemDataset : public Dataset { std::shared_ptr format, std::vector> fragments); - /// \brief Write to a new format and filesystem location, preserving partitioning. + /// \brief Write a dataset. /// - /// \param[in] plan the WritePlan to execute. - /// \param[in] scan_options options in which to scan fragments - /// \param[in] scan_context context in which to scan fragments before writing. - static Result> Write( - const WritePlan& plan, std::shared_ptr scan_options, - std::shared_ptr scan_context); + /// \param[in] schema Schema of written dataset. + /// \param[in] format FileFormat with which fragments will be written. + /// \param[in] filesystem FileSystem into which the dataset will be written. + /// \param[in] base_dir Root directory into which the dataset will be written. + /// \param[in] partitioning Partitioning used to generate fragment paths. + /// \param[in] scan_context Resource pool used to scan and write fragments. + /// \param[in] fragments Fragments to be written to disk. + static Status Write(std::shared_ptr schema, std::shared_ptr format, + std::shared_ptr filesystem, std::string base_dir, + std::shared_ptr partitioning, + std::shared_ptr scan_context, + FragmentIterator fragments); /// \brief Return the type name of the dataset. std::string type_name() const override { return "filesystem"; } @@ -284,69 +248,5 @@ class ARROW_DS_EXPORT FileSystemDataset : public Dataset { std::vector> fragments_; }; -/// \brief Write a fragment to a single OutputStream. -class ARROW_DS_EXPORT WriteTask { - public: - virtual Status Execute() = 0; - - virtual ~WriteTask() = default; - - const WritableFileSource& destination() const; - const std::shared_ptr& format() const { return format_; } - - protected: - WriteTask(WritableFileSource destination, std::shared_ptr format) - : destination_(std::move(destination)), format_(std::move(format)) {} - - Status CreateDestinationParentDir() const; - - WritableFileSource destination_; - std::shared_ptr format_; -}; - -/// \brief A declarative plan for writing fragments to a partitioned directory structure. -class ARROW_DS_EXPORT WritePlan { - public: - /// The partitioning with which paths were generated - std::shared_ptr partitioning; - - /// The schema of the Dataset which will be written - std::shared_ptr schema; - - /// The format into which fragments will be written - std::shared_ptr format; - - /// The FileSystem and base directory for partitioned writing - std::shared_ptr filesystem; - std::string partition_base_dir; - - class FragmentOrPartitionExpression { - public: - enum Kind { EXPRESSION, FRAGMENT }; - - explicit FragmentOrPartitionExpression(std::shared_ptr partition_expr) - : kind_(EXPRESSION), partition_expr_(std::move(partition_expr)) {} - - explicit FragmentOrPartitionExpression(std::shared_ptr fragment) - : kind_(FRAGMENT), fragment_(std::move(fragment)) {} - - Kind kind() const { return kind_; } - - const std::shared_ptr& partition_expr() const { return partition_expr_; } - const std::shared_ptr& fragment() const { return fragment_; } - - private: - Kind kind_; - std::shared_ptr partition_expr_; - std::shared_ptr fragment_; - }; - - /// If fragment_or_partition_expressions[i] is a Fragment, that Fragment will be - /// written to paths[i]. If it is an Expression, a directory representing that partition - /// expression will be created at paths[i] instead. - std::vector fragment_or_partition_expressions; - std::vector paths; -}; - } // namespace dataset } // namespace arrow diff --git a/cpp/src/arrow/dataset/file_ipc.cc b/cpp/src/arrow/dataset/file_ipc.cc index 63f30204c02e..de25875b59a6 100644 --- a/cpp/src/arrow/dataset/file_ipc.cc +++ b/cpp/src/arrow/dataset/file_ipc.cc @@ -159,54 +159,17 @@ Result IpcFileFormat::ScanFile(std::shared_ptr op fragment->source()); } -class IpcWriteTask : public WriteTask { - public: - IpcWriteTask(WritableFileSource destination, std::shared_ptr format, - std::shared_ptr fragment, - std::shared_ptr scan_options, - std::shared_ptr scan_context) - : WriteTask(std::move(destination), std::move(format)), - fragment_(std::move(fragment)), - scan_options_(std::move(scan_options)), - scan_context_(std::move(scan_context)) {} - - Status Execute() override { - RETURN_NOT_OK(CreateDestinationParentDir()); - - auto schema = scan_options_->schema(); - - ARROW_ASSIGN_OR_RAISE(auto out_stream, destination_.Open()); - ARROW_ASSIGN_OR_RAISE(auto writer, ipc::NewFileWriter(out_stream.get(), schema)); - ARROW_ASSIGN_OR_RAISE(auto scan_task_it, - fragment_->Scan(scan_options_, scan_context_)); - - for (auto maybe_scan_task : scan_task_it) { - ARROW_ASSIGN_OR_RAISE(auto scan_task, maybe_scan_task); - - ARROW_ASSIGN_OR_RAISE(auto batch_it, scan_task->Execute()); - - for (auto maybe_batch : batch_it) { - ARROW_ASSIGN_OR_RAISE(auto batch, std::move(maybe_batch)); - RETURN_NOT_OK(writer->WriteRecordBatch(*batch)); - } - } - - return writer->Close(); +Status IpcFileFormat::WriteFragment(RecordBatchReader* batches, + io::OutputStream* destination) { + ARROW_ASSIGN_OR_RAISE(auto writer, ipc::NewFileWriter(destination, batches->schema())); + + for (;;) { + ARROW_ASSIGN_OR_RAISE(auto batch, batches->Next()); + if (batch == nullptr) break; + RETURN_NOT_OK(writer->WriteRecordBatch(*batch)); } - private: - std::shared_ptr fragment_; - std::shared_ptr scan_options_; - std::shared_ptr scan_context_; -}; - -Result> IpcFileFormat::WriteFragment( - WritableFileSource destination, std::shared_ptr fragment, - std::shared_ptr scan_options, - std::shared_ptr scan_context) { - return std::make_shared(std::move(destination), shared_from_this(), - std::move(fragment), std::move(scan_options), - std::move(scan_context)); + return writer->Close(); } } // namespace dataset diff --git a/cpp/src/arrow/dataset/file_ipc.h b/cpp/src/arrow/dataset/file_ipc.h index ad18354b301e..2b8b94de9845 100644 --- a/cpp/src/arrow/dataset/file_ipc.h +++ b/cpp/src/arrow/dataset/file_ipc.h @@ -47,10 +47,8 @@ class ARROW_DS_EXPORT IpcFileFormat : public FileFormat { std::shared_ptr context, FileFragment* fragment) const override; - Result> WriteFragment( - WritableFileSource destination, std::shared_ptr fragment, - std::shared_ptr options, - std::shared_ptr context) override; + Status WriteFragment(RecordBatchReader* batches, + io::OutputStream* destination) override; }; } // namespace dataset diff --git a/cpp/src/arrow/dataset/file_ipc_test.cc b/cpp/src/arrow/dataset/file_ipc_test.cc index c557621d3940..e1b574095bbf 100644 --- a/cpp/src/arrow/dataset/file_ipc_test.cc +++ b/cpp/src/arrow/dataset/file_ipc_test.cc @@ -22,6 +22,7 @@ #include #include "arrow/dataset/dataset_internal.h" +#include "arrow/dataset/discovery.h" #include "arrow/dataset/file_base.h" #include "arrow/dataset/filter.h" #include "arrow/dataset/partition.h" @@ -87,10 +88,10 @@ class TestIpcFileFormat : public ArrowIpcWriterMixin { kBatchRepetitions); } - Result GetFileSink() { + Result> GetFileSink() { ARROW_ASSIGN_OR_RAISE(std::shared_ptr buffer, AllocateResizableBuffer(0)); - return WritableFileSource(std::move(buffer)); + return std::make_shared(buffer); } RecordBatchIterator Batches(ScanTaskIterator scan_task_it) { @@ -149,63 +150,327 @@ TEST_F(TestIpcFileFormat, ScanRecordBatchReaderWithVirtualColumn) { TEST_F(TestIpcFileFormat, WriteRecordBatchReader) { std::shared_ptr reader = GetRecordBatchReader(); auto source = GetFileSource(reader.get()); + reader = GetRecordBatchReader(); opts_ = ScanOptions::Make(reader->schema()); - ASSERT_OK_AND_ASSIGN(auto fragment, format_->MakeFragment(*source)); EXPECT_OK_AND_ASSIGN(auto sink, GetFileSink()); - EXPECT_OK_AND_ASSIGN(auto write_task, - format_->WriteFragment(sink, fragment, opts_, ctx_)); + ASSERT_OK(format_->WriteFragment(reader.get(), sink.get())); - ASSERT_OK(write_task->Execute()); + EXPECT_OK_AND_ASSIGN(auto written, sink->Finish()); - AssertBufferEqual(*sink.buffer(), *source->buffer()); + AssertBufferEqual(*written, *source->buffer()); } class TestIpcFileSystemDataset : public TestIpcFileFormat, - public MakeFileSystemDatasetMixin {}; - -TEST_F(TestIpcFileSystemDataset, Write) { - std::string paths = R"( - old_root/i32=0/str=aaa/dat - old_root/i32=0/str=bbb/dat - old_root/i32=0/str=ccc/dat - old_root/i32=1/str=aaa/dat - old_root/i32=1/str=bbb/dat - old_root/i32=1/str=ccc/dat - )"; - - ExpressionVector partitions{ - ("i32"_ == 0 and "str"_ == "aaa").Copy(), ("i32"_ == 0 and "str"_ == "bbb").Copy(), - ("i32"_ == 0 and "str"_ == "ccc").Copy(), ("i32"_ == 1 and "str"_ == "aaa").Copy(), - ("i32"_ == 1 and "str"_ == "bbb").Copy(), ("i32"_ == 1 and "str"_ == "ccc").Copy(), - }; - - MakeDatasetFromPathlist(paths, scalar(true), partitions); - - auto schema = arrow::schema({field("i32", int32()), field("str", utf8())}); - opts_ = ScanOptions::Make(schema); - - auto partitioning_factory = DirectoryPartitioning::MakeFactory({"str", "i32"}); - ASSERT_OK_AND_ASSIGN( - auto plan, partitioning_factory->MakeWritePlan(schema, dataset_->GetFragments())); + public MakeFileSystemDatasetMixin { + public: + using PathAndContent = std::unordered_map; + + void SetUp() override { + PathAndContent source_files; + + source_files["/dataset/year=2018/month=01/dat0.json"] = R"([ + {"region": "NY", "model": "3", "sales": 742.0, "country": "US"}, + {"region": "NY", "model": "S", "sales": 304.125, "country": "US"}, + {"region": "NY", "model": "Y", "sales": 27.5, "country": "US"} + ])"; + source_files["/dataset/year=2018/month=01/dat1.json"] = R"([ + {"region": "QC", "model": "3", "sales": 512, "country": "CA"}, + {"region": "QC", "model": "S", "sales": 978, "country": "CA"}, + {"region": "NY", "model": "X", "sales": 136.25, "country": "US"}, + {"region": "QC", "model": "X", "sales": 1.0, "country": "CA"}, + {"region": "QC", "model": "Y", "sales": 69, "country": "CA"} + ])"; + source_files["/dataset/year=2019/month=01/dat0.json"] = R"([ + {"region": "CA", "model": "3", "sales": 273.5, "country": "US"}, + {"region": "CA", "model": "S", "sales": 13, "country": "US"}, + {"region": "CA", "model": "X", "sales": 54, "country": "US"}, + {"region": "QC", "model": "S", "sales": 10, "country": "CA"}, + {"region": "CA", "model": "Y", "sales": 21, "country": "US"} + ])"; + source_files["/dataset/year=2019/month=01/dat1.json"] = R"([ + {"region": "QC", "model": "3", "sales": 152.25, "country": "CA"}, + {"region": "QC", "model": "X", "sales": 42, "country": "CA"}, + {"region": "QC", "model": "Y", "sales": 37, "country": "CA"} + ])"; + source_files["/dataset/.pesky"] = "garbage content"; + + auto mock_fs = std::make_shared(fs::kNoTime); + for (const auto& f : source_files) { + ARROW_EXPECT_OK(mock_fs->CreateFile(f.first, f.second, /* recursive */ true)); + } + fs_ = mock_fs; + + /// schema for the whole dataset (both source and destination) + schema_ = schema({ + field("region", utf8()), + field("model", utf8()), + field("sales", float64()), + field("year", int32()), + field("month", int32()), + field("country", utf8()), + }); + + /// Dummy file format for source dataset. Note that it isn't partitioned on country + auto source_format = std::make_shared( + SchemaFromColumnNames(schema_, {"region", "model", "sales", "country"})); + + fs::FileSelector s; + s.base_dir = "/dataset"; + s.recursive = true; + + FileSystemFactoryOptions options; + options.selector_ignore_prefixes = {"."}; + options.partitioning = HivePartitioning::MakeFactory(); + ASSERT_OK_AND_ASSIGN(auto factory, + FileSystemDatasetFactory::Make(fs_, s, source_format, options)); + ASSERT_OK_AND_ASSIGN(dataset_, factory->Finish()); + } + + void AssertWrittenAsExpected() { + std::vector files; + for (const auto& file_contents : expected_files_) { + files.push_back(file_contents.first); + } + EXPECT_THAT(checked_pointer_cast(written_)->files(), + testing::UnorderedElementsAreArray(files)); + + for (auto maybe_fragment : written_->GetFragments()) { + ASSERT_OK_AND_ASSIGN(auto fragment, std::move(maybe_fragment)); + + ASSERT_OK_AND_ASSIGN(auto actual_physical_schema, fragment->ReadPhysicalSchema()); + AssertSchemaEqual(*expected_physical_schema_, *actual_physical_schema, + /*verbose=*/true); + + const auto& path = checked_pointer_cast(fragment)->source().path(); - plan.format = format_; - plan.filesystem = fs_; - plan.partition_base_dir = "new_root/"; + auto expected_struct = ArrayFromJSON(struct_(expected_physical_schema_->fields()), + {expected_files_[path]}); - ASSERT_OK_AND_ASSIGN(auto written, FileSystemDataset::Write(plan, opts_, ctx_)); + ASSERT_OK_AND_ASSIGN(auto scanner, ScannerBuilder(actual_physical_schema, fragment, + std::make_shared()) + .Finish()); + ASSERT_OK_AND_ASSIGN(auto actual_table, scanner->ToTable()); + ASSERT_OK_AND_ASSIGN(actual_table, actual_table->CombineChunks()); + std::shared_ptr actual_struct; - auto parent_directories = written->files(); - for (auto& path : parent_directories) { - EXPECT_EQ(fs::internal::GetAbstractPathExtension(path), "ipc"); - path = fs::internal::GetAbstractPathParent(path).first; + for (auto maybe_batch : + IteratorFromReader(std::make_shared(*actual_table))) { + ASSERT_OK_AND_ASSIGN(auto batch, std::move(maybe_batch)); + ASSERT_OK_AND_ASSIGN(actual_struct, batch->ToStructArray()); + } + + AssertArraysEqual(*expected_struct, *actual_struct, /*verbose=*/true); + } } - EXPECT_THAT(parent_directories, - testing::ElementsAre("new_root/aaa/0", "new_root/aaa/1", "new_root/bbb/0", - "new_root/bbb/1", "new_root/ccc/0", "new_root/ccc/1")); + PathAndContent expected_files_; + std::shared_ptr expected_physical_schema_; + std::shared_ptr written_; +}; + +TEST_F(TestIpcFileSystemDataset, WriteWithIdenticalPartitioningSchema) { + auto desired_partitioning = std::make_shared( + SchemaFromColumnNames(schema_, {"year", "month"})); + + ASSERT_OK(FileSystemDataset::Write( + schema_, format_, fs_, "new_root/", desired_partitioning, + std::make_shared(), dataset_->GetFragments())); + + fs::FileSelector s; + s.recursive = true; + s.base_dir = "/new_root"; + + FileSystemFactoryOptions options; + options.partitioning = desired_partitioning; + ASSERT_OK_AND_ASSIGN(auto factory, + FileSystemDatasetFactory::Make(fs_, s, format_, options)); + ASSERT_OK_AND_ASSIGN(written_, factory->Finish()); + + expected_files_["/new_root/2018/1/dat_0.ipc"] = R"([ + {"region": "NY", "model": "3", "sales": 742.0, "country": "US"}, + {"region": "NY", "model": "S", "sales": 304.125, "country": "US"}, + {"region": "NY", "model": "Y", "sales": 27.5, "country": "US"} + ])"; + expected_files_["/new_root/2018/1/dat_1.ipc"] = R"([ + {"region": "QC", "model": "3", "sales": 512, "country": "CA"}, + {"region": "QC", "model": "S", "sales": 978, "country": "CA"}, + {"region": "NY", "model": "X", "sales": 136.25, "country": "US"}, + {"region": "QC", "model": "X", "sales": 1.0, "country": "CA"}, + {"region": "QC", "model": "Y", "sales": 69, "country": "CA"} + ])"; + expected_files_["/new_root/2019/1/dat_2.ipc"] = R"([ + {"region": "CA", "model": "3", "sales": 273.5, "country": "US"}, + {"region": "CA", "model": "S", "sales": 13, "country": "US"}, + {"region": "CA", "model": "X", "sales": 54, "country": "US"}, + {"region": "QC", "model": "S", "sales": 10, "country": "CA"}, + {"region": "CA", "model": "Y", "sales": 21, "country": "US"} + ])"; + expected_files_["/new_root/2019/1/dat_3.ipc"] = R"([ + {"region": "QC", "model": "3", "sales": 152.25, "country": "CA"}, + {"region": "QC", "model": "X", "sales": 42, "country": "CA"}, + {"region": "QC", "model": "Y", "sales": 37, "country": "CA"} + ])"; + expected_physical_schema_ = + SchemaFromColumnNames(schema_, {"region", "model", "sales", "country"}); + + AssertWrittenAsExpected(); +} + +TEST_F(TestIpcFileSystemDataset, WriteWithUnrelatedPartitioningSchema) { + auto desired_partitioning = std::make_shared( + SchemaFromColumnNames(schema_, {"country", "region"})); + + ASSERT_OK(FileSystemDataset::Write( + schema_, format_, fs_, "new_root/", desired_partitioning, + std::make_shared(), dataset_->GetFragments())); + + fs::FileSelector s; + s.recursive = true; + s.base_dir = "/new_root"; + + FileSystemFactoryOptions options; + options.partitioning = desired_partitioning; + ASSERT_OK_AND_ASSIGN(auto factory, + FileSystemDatasetFactory::Make(fs_, s, format_, options)); + ASSERT_OK_AND_ASSIGN(written_, factory->Finish()); + + // XXX first thing a user will be annoyed by: we don't support left + // padding the month field with 0. + expected_files_["/new_root/US/NY/dat_0.ipc"] = R"([ + {"year": 2018, "month": 1, "model": "3", "sales": 742.0}, + {"year": 2018, "month": 1, "model": "S", "sales": 304.125}, + {"year": 2018, "month": 1, "model": "Y", "sales": 27.5} + ])"; + expected_files_["/new_root/US/NY/dat_1.ipc"] = R"([ + {"year": 2018, "month": 1, "model": "X", "sales": 136.25} + ])"; + expected_files_["/new_root/CA/QC/dat_1.ipc"] = R"([ + {"year": 2018, "month": 1, "model": "3", "sales": 512}, + {"year": 2018, "month": 1, "model": "S", "sales": 978}, + {"year": 2018, "month": 1, "model": "X", "sales": 1.0}, + {"year": 2018, "month": 1, "model": "Y", "sales": 69} + ])"; + expected_files_["/new_root/US/CA/dat_2.ipc"] = R"([ + {"year": 2019, "month": 1, "model": "3", "sales": 273.5}, + {"year": 2019, "month": 1, "model": "S", "sales": 13}, + {"year": 2019, "month": 1, "model": "X", "sales": 54}, + {"year": 2019, "month": 1, "model": "Y", "sales": 21} + ])"; + expected_files_["/new_root/CA/QC/dat_2.ipc"] = R"([ + {"year": 2019, "month": 1, "model": "S", "sales": 10} + ])"; + expected_files_["/new_root/CA/QC/dat_3.ipc"] = R"([ + {"year": 2019, "month": 1, "model": "3", "sales": 152.25}, + {"year": 2019, "month": 1, "model": "X", "sales": 42}, + {"year": 2019, "month": 1, "model": "Y", "sales": 37} + ])"; + expected_physical_schema_ = + SchemaFromColumnNames(schema_, {"model", "sales", "year", "month"}); + + AssertWrittenAsExpected(); +} + +TEST_F(TestIpcFileSystemDataset, WriteWithSupersetPartitioningSchema) { + auto desired_partitioning = std::make_shared( + SchemaFromColumnNames(schema_, {"year", "month", "country", "region"})); + + ASSERT_OK(FileSystemDataset::Write( + schema_, format_, fs_, "new_root/", desired_partitioning, + std::make_shared(), dataset_->GetFragments())); + + fs::FileSelector s; + s.recursive = true; + s.base_dir = "/new_root"; + + FileSystemFactoryOptions options; + options.partitioning = desired_partitioning; + ASSERT_OK_AND_ASSIGN(auto factory, + FileSystemDatasetFactory::Make(fs_, s, format_, options)); + ASSERT_OK_AND_ASSIGN(written_, factory->Finish()); + + // XXX first thing a user will be annoyed by: we don't support left + // padding the month field with 0. + expected_files_["/new_root/2018/1/US/NY/dat_0.ipc"] = R"([ + {"model": "3", "sales": 742.0}, + {"model": "S", "sales": 304.125}, + {"model": "Y", "sales": 27.5} + ])"; + expected_files_["/new_root/2018/1/US/NY/dat_1.ipc"] = R"([ + {"model": "X", "sales": 136.25} + ])"; + expected_files_["/new_root/2018/1/CA/QC/dat_1.ipc"] = R"([ + {"model": "3", "sales": 512}, + {"model": "S", "sales": 978}, + {"model": "X", "sales": 1.0}, + {"model": "Y", "sales": 69} + ])"; + expected_files_["/new_root/2019/1/US/CA/dat_2.ipc"] = R"([ + {"model": "3", "sales": 273.5}, + {"model": "S", "sales": 13}, + {"model": "X", "sales": 54}, + {"model": "Y", "sales": 21} + ])"; + expected_files_["/new_root/2019/1/CA/QC/dat_2.ipc"] = R"([ + {"model": "S", "sales": 10} + ])"; + expected_files_["/new_root/2019/1/CA/QC/dat_3.ipc"] = R"([ + {"model": "3", "sales": 152.25}, + {"model": "X", "sales": 42}, + {"model": "Y", "sales": 37} + ])"; + expected_physical_schema_ = SchemaFromColumnNames(schema_, {"model", "sales"}); + + AssertWrittenAsExpected(); +} + +TEST_F(TestIpcFileSystemDataset, WriteWithEmptyPartitioningSchema) { + auto desired_partitioning = + std::make_shared(SchemaFromColumnNames(schema_, {})); + + ASSERT_OK(FileSystemDataset::Write( + schema_, format_, fs_, "new_root/", desired_partitioning, + std::make_shared(), dataset_->GetFragments())); + + fs::FileSelector s; + s.recursive = true; + s.base_dir = "/new_root"; + + FileSystemFactoryOptions options; + options.partitioning = desired_partitioning; + ASSERT_OK_AND_ASSIGN(auto factory, + FileSystemDatasetFactory::Make(fs_, s, format_, options)); + ASSERT_OK_AND_ASSIGN(written_, factory->Finish()); + + expected_files_["/new_root/dat_0.ipc"] = R"([ + {"country": "US", "region": "NY", "year": 2018, "month": 1, "model": "3", "sales": 742.0}, + {"country": "US", "region": "NY", "year": 2018, "month": 1, "model": "S", "sales": 304.125}, + {"country": "US", "region": "NY", "year": 2018, "month": 1, "model": "Y", "sales": 27.5} + ])"; + expected_files_["/new_root/dat_1.ipc"] = R"([ + {"country": "CA", "region": "QC", "year": 2018, "month": 1, "model": "3", "sales": 512}, + {"country": "CA", "region": "QC", "year": 2018, "month": 1, "model": "S", "sales": 978}, + {"country": "US", "region": "NY", "year": 2018, "month": 1, "model": "X", "sales": 136.25}, + {"country": "CA", "region": "QC", "year": 2018, "month": 1, "model": "X", "sales": 1.0}, + {"country": "CA", "region": "QC", "year": 2018, "month": 1, "model": "Y", "sales": 69} + ])"; + expected_files_["/new_root/dat_2.ipc"] = R"([ + {"country": "US", "region": "CA", "year": 2019, "month": 1, "model": "3", "sales": 273.5}, + {"country": "US", "region": "CA", "year": 2019, "month": 1, "model": "S", "sales": 13}, + {"country": "US", "region": "CA", "year": 2019, "month": 1, "model": "X", "sales": 54}, + {"country": "CA", "region": "QC", "year": 2019, "month": 1, "model": "S", "sales": 10}, + {"country": "US", "region": "CA", "year": 2019, "month": 1, "model": "Y", "sales": 21} + ])"; + expected_files_["/new_root/dat_3.ipc"] = R"([ + {"country": "CA", "region": "QC", "year": 2019, "month": 1, "model": "3", "sales": 152.25}, + {"country": "CA", "region": "QC", "year": 2019, "month": 1, "model": "X", "sales": 42}, + {"country": "CA", "region": "QC", "year": 2019, "month": 1, "model": "Y", "sales": 37} + ])"; + expected_physical_schema_ = schema_; + + AssertWrittenAsExpected(); } TEST_F(TestIpcFileFormat, OpenFailureWithRelevantError) { diff --git a/cpp/src/arrow/dataset/filter.cc b/cpp/src/arrow/dataset/filter.cc index b35b8da2bb70..d99d624b91d0 100644 --- a/cpp/src/arrow/dataset/filter.cc +++ b/cpp/src/arrow/dataset/filter.cc @@ -28,6 +28,7 @@ #include "arrow/buffer.h" #include "arrow/buffer_builder.h" +#include "arrow/builder.h" #include "arrow/compute/api.h" #include "arrow/dataset/dataset.h" #include "arrow/io/memory.h" @@ -38,6 +39,7 @@ #include "arrow/scalar.h" #include "arrow/type_fwd.h" #include "arrow/util/checked_cast.h" +#include "arrow/util/int_util.h" #include "arrow/util/iterator.h" #include "arrow/util/logging.h" #include "arrow/util/string.h" @@ -1493,5 +1495,191 @@ Result> Expression::Deserialize(const Buffer& serial return DeserializeImpl{}.FromBuffer(serialized); } +// Transform an array of counts to offsets which will divide a ListArray +// into an equal number of slices with corresponding lengths. +inline Result> CountsToOffsets( + std::shared_ptr counts) { + Int32Builder offset_builder; + RETURN_NOT_OK(offset_builder.Resize(counts->length() + 1)); + offset_builder.UnsafeAppend(0); + + for (int64_t i = 0; i < counts->length(); ++i) { + DCHECK_NE(counts->Value(i), 0); + auto next_offset = static_cast(offset_builder[i] + counts->Value(i)); + offset_builder.UnsafeAppend(next_offset); + } + + std::shared_ptr offsets; + RETURN_NOT_OK(offset_builder.Finish(&offsets)); + return offsets; +} + +// Helper for simultaneous dictionary encoding of multiple arrays. +// +// The fused dictionary is the Cartesian product of the individual dictionaries. +// For example given two arrays A, B where A has unique values ["ex", "why"] +// and B has unique values [0, 1] the fused dictionary is the set of tuples +// [["ex", 0], ["ex", 1], ["why", 0], ["ex", 1]]. +// +// TODO(bkietz) this capability belongs in an Action of the hash kernels, where +// it can be used to group aggregates without materializing a grouped batch. +// For the purposes of writing we need the materialized grouped batch anyway +// since no Writers accept a selection vector. +class StructDictionary { + public: + struct Encoded { + std::shared_ptr indices; + std::shared_ptr dictionary; + }; + + static Result Encode(const ArrayVector& columns) { + Encoded out{nullptr, std::make_shared()}; + + for (const auto& column : columns) { + if (column->null_count() != 0) { + return Status::NotImplemented("Grouping on a field with nulls"); + } + + RETURN_NOT_OK(out.dictionary->AddOne(column, &out.indices)); + } + + return out; + } + + Result> Decode(std::shared_ptr fused_indices, + FieldVector fields) { + std::vector builders(dictionaries_.size()); + for (Int32Builder& b : builders) { + RETURN_NOT_OK(b.Resize(fused_indices->length())); + } + + std::vector codes(dictionaries_.size()); + for (int64_t i = 0; i < fused_indices->length(); ++i) { + Expand(fused_indices->Value(i), codes.data()); + + auto builder_it = builders.begin(); + for (int32_t index : codes) { + builder_it++->UnsafeAppend(index); + } + } + + ArrayVector columns(dictionaries_.size()); + for (size_t i = 0; i < dictionaries_.size(); ++i) { + std::shared_ptr indices; + RETURN_NOT_OK(builders[i].FinishInternal(&indices)); + + ARROW_ASSIGN_OR_RAISE(Datum column, compute::Take(dictionaries_[i], indices)); + columns[i] = column.make_array(); + } + + return StructArray::Make(std::move(columns), std::move(fields)); + } + + private: + Status AddOne(const std::shared_ptr& column, + std::shared_ptr* fused_indices) { + ARROW_ASSIGN_OR_RAISE(Datum encoded, compute::DictionaryEncode(column)); + ArrayData* encoded_array = encoded.mutable_array(); + + auto indices = std::make_shared(encoded_array->length, + std::move(encoded_array->buffers[1])); + + dictionaries_.push_back(MakeArray(std::move(encoded_array->dictionary))); + auto dictionary_size = static_cast(dictionaries_.back()->length()); + + if (*fused_indices == nullptr) { + *fused_indices = std::move(indices); + size_ = dictionary_size; + return Status::OK(); + } + + // It's useful to think about the case where each of dictionaries_ has size 10. + // In this case the decimal digit in the ones place is the code in dictionaries_[0], + // the tens place corresponds to dictionaries_[1], etc. + // The incumbent indices must be shifted to the hundreds place so as not to collide. + ARROW_ASSIGN_OR_RAISE(Datum new_fused_indices, + compute::Multiply(indices, MakeScalar(size_))); + + ARROW_ASSIGN_OR_RAISE(new_fused_indices, + compute::Add(new_fused_indices, *fused_indices)); + + *fused_indices = checked_pointer_cast(new_fused_indices.make_array()); + + // XXX should probably cap this at 2**15 or so + DCHECK(!internal::HasPositiveMultiplyOverflow(size_, dictionary_size)); + size_ *= dictionary_size; + return Status::OK(); + } + + // expand a fused code into component dict codes, order is in order of addition + void Expand(int32_t fused_code, int32_t* codes) { + for (size_t i = 0; i < dictionaries_.size(); ++i) { + auto dictionary_size = static_cast(dictionaries_[i]->length()); + codes[i] = fused_code % dictionary_size; + fused_code /= dictionary_size; + } + } + + int32_t size_; + ArrayVector dictionaries_; +}; + +Result> MakeGroupings(const StructArray& by) { + if (by.num_fields() == 0) { + return Status::NotImplemented("Grouping with no criteria"); + } + + ARROW_ASSIGN_OR_RAISE(auto fused, StructDictionary::Encode(by.fields())); + + ARROW_ASSIGN_OR_RAISE(auto sort_indices, compute::SortToIndices(*fused.indices)); + ARROW_ASSIGN_OR_RAISE(Datum sorted, compute::Take(fused.indices, *sort_indices)); + fused.indices = checked_pointer_cast(sorted.make_array()); + + ARROW_ASSIGN_OR_RAISE(auto fused_counts_and_values, + compute::ValueCounts(fused.indices)); + fused.indices.reset(); + + auto unique_fused_indices = + checked_pointer_cast(fused_counts_and_values->GetFieldByName("values")); + ARROW_ASSIGN_OR_RAISE( + auto unique_rows, + fused.dictionary->Decode(std::move(unique_fused_indices), by.type()->fields())); + + auto counts = + checked_pointer_cast(fused_counts_and_values->GetFieldByName("counts")); + ARROW_ASSIGN_OR_RAISE(auto offsets, CountsToOffsets(std::move(counts))); + + ARROW_ASSIGN_OR_RAISE(auto grouped_sort_indices, + ListArray::FromArrays(*offsets, *sort_indices)); + + return StructArray::Make( + ArrayVector{std::move(unique_rows), std::move(grouped_sort_indices)}, + std::vector{"values", "groupings"}); +} + +Result> ApplyGroupings(const ListArray& groupings, + const Array& array) { + ARROW_ASSIGN_OR_RAISE(Datum sorted, + compute::Take(array, groupings.data()->child_data[0])); + + return std::make_shared(list(array.type()), groupings.length(), + groupings.value_offsets(), sorted.make_array()); +} + +Result ApplyGroupings(const ListArray& groupings, + const std::shared_ptr& batch) { + ARROW_ASSIGN_OR_RAISE(Datum sorted, + compute::Take(batch, groupings.data()->child_data[0])); + + const auto& sorted_batch = *sorted.record_batch(); + + RecordBatchVector out(static_cast(groupings.length())); + for (size_t i = 0; i < out.size(); ++i) { + out[i] = sorted_batch.Slice(groupings.value_offset(i), groupings.value_length(i)); + } + + return out; +} + } // namespace dataset } // namespace arrow diff --git a/cpp/src/arrow/dataset/filter.h b/cpp/src/arrow/dataset/filter.h index b7d4655dbcfd..ebf58cc6996d 100644 --- a/cpp/src/arrow/dataset/filter.h +++ b/cpp/src/arrow/dataset/filter.h @@ -641,5 +641,22 @@ class ARROW_DS_EXPORT TreeEvaluator : public ExpressionEvaluator { struct Impl; }; +/// \brief Assemble lists of indices of identical rows. +/// +/// \param[in] by A StructArray whose columns will be used as grouping criteria. +/// \return A StructArray mapping unique rows (in field "values", represented as a +/// StructArray with the same fields as `by`) to lists of indices where +/// that row appears (in field "groupings"). +ARROW_DS_EXPORT +Result> MakeGroupings(const StructArray& by); + +/// \brief Produce slices of an Array which correspond to the provided groupings. +ARROW_DS_EXPORT +Result> ApplyGroupings(const ListArray& groupings, + const Array& array); +ARROW_DS_EXPORT +Result ApplyGroupings(const ListArray& groupings, + const std::shared_ptr& batch); + } // namespace dataset } // namespace arrow diff --git a/cpp/src/arrow/dataset/filter_test.cc b/cpp/src/arrow/dataset/filter_test.cc index 7c3c1a28588b..8e16208cb80e 100644 --- a/cpp/src/arrow/dataset/filter_test.cc +++ b/cpp/src/arrow/dataset/filter_test.cc @@ -592,5 +592,57 @@ TEST(ExpressionSerializationTest, RoundTrips) { } } +void AssertGrouping(const FieldVector& by_fields, const std::string& batch_json, + const std::string& expected_json) { + FieldVector fields_with_ids = by_fields; + fields_with_ids.push_back(field("ids", list(int32()))); + auto expected = ArrayFromJSON(struct_(fields_with_ids), expected_json); + + FieldVector fields_with_id = by_fields; + fields_with_id.push_back(field("id", int32())); + auto batch = RecordBatchFromJSON(schema(fields_with_id), batch_json); + + ASSERT_OK_AND_ASSIGN(auto by, batch->RemoveColumn(batch->num_columns() - 1) + .Map([](std::shared_ptr by) { + return by->ToStructArray(); + })); + + ASSERT_OK_AND_ASSIGN(auto groupings_and_values, MakeGroupings(*by)); + + auto groupings = + checked_pointer_cast(groupings_and_values->GetFieldByName("groupings")); + + ASSERT_OK_AND_ASSIGN(std::shared_ptr grouped_ids, + ApplyGroupings(*groupings, *batch->GetColumnByName("id"))); + + ArrayVector columns = + checked_cast(*groupings_and_values->GetFieldByName("values")) + .fields(); + columns.push_back(grouped_ids); + + ASSERT_OK_AND_ASSIGN(auto actual, StructArray::Make(columns, fields_with_ids)); + + AssertArraysEqual(*expected, *actual, /*verbose=*/true); +} + +TEST(GroupTest, Basics) { + AssertGrouping({field("a", utf8()), field("b", int32())}, R"([ + {"a": "ex", "b": 0, "id": 0}, + {"a": "ex", "b": 0, "id": 1}, + {"a": "why", "b": 0, "id": 2}, + {"a": "ex", "b": 1, "id": 3}, + {"a": "why", "b": 0, "id": 4}, + {"a": "ex", "b": 1, "id": 5}, + {"a": "ex", "b": 0, "id": 6}, + {"a": "why", "b": 1, "id": 7} + ])", + R"([ + {"a": "ex", "b": 0, "ids": [0, 1, 6]}, + {"a": "why", "b": 0, "ids": [2, 4]}, + {"a": "ex", "b": 1, "ids": [3, 5]}, + {"a": "why", "b": 1, "ids": [7]} + ])"); +} + } // namespace dataset } // namespace arrow diff --git a/cpp/src/arrow/dataset/partition.cc b/cpp/src/arrow/dataset/partition.cc index 5844f89c87c6..a0ea91def1d8 100644 --- a/cpp/src/arrow/dataset/partition.cc +++ b/cpp/src/arrow/dataset/partition.cc @@ -26,26 +26,30 @@ #include #include "arrow/array/array_base.h" +#include "arrow/array/array_nested.h" #include "arrow/array/builder_binary.h" #include "arrow/compute/api_scalar.h" #include "arrow/dataset/dataset_internal.h" #include "arrow/dataset/file_base.h" #include "arrow/dataset/filter.h" #include "arrow/dataset/scanner.h" +#include "arrow/dataset/scanner_internal.h" #include "arrow/filesystem/filesystem.h" #include "arrow/filesystem/path_util.h" #include "arrow/scalar.h" #include "arrow/util/iterator.h" +#include "arrow/util/logging.h" #include "arrow/util/range.h" #include "arrow/util/sort.h" #include "arrow/util/string_view.h" namespace arrow { -namespace dataset { +using internal::checked_cast; +using internal::checked_pointer_cast; using util::string_view; -using arrow::internal::checked_cast; +namespace dataset { std::shared_ptr Partitioning::Default() { class DefaultPartitioning : public Partitioning { @@ -62,24 +66,16 @@ std::shared_ptr Partitioning::Default() { return Status::NotImplemented("formatting paths from ", type_name(), " Partitioning"); } + + Result> Partition( + const std::shared_ptr& batch) const override { + return std::vector{{batch, scalar(true)}}; + } }; return std::make_shared(); } -Result PartitioningFactory::MakeWritePlan(std::shared_ptr schema, - FragmentIterator fragment_it) { - return Status::NotImplemented("MakeWritePlan from PartitioningFactory of type ", - type_name()); -} - -Result PartitioningFactory::MakeWritePlan( - std::shared_ptr schema, FragmentIterator fragment_it, - std::shared_ptr partition_schema) { - return Status::NotImplemented("MakeWritePlan from PartitioningFactory of type ", - type_name()); -} - Status KeyValuePartitioning::VisitKeys( const Expression& expr, const std::function ConjunctionFromGroupingRow(Scalar* row) { + ScalarVector* values = &checked_cast(row)->value; + ExpressionVector equality_expressions(values->size()); + for (size_t i = 0; i < values->size(); ++i) { + const std::string& name = row->type->field(static_cast(i))->name(); + equality_expressions[i] = equal(field_ref(name), scalar(std::move(values->at(i)))); + } + return and_(std::move(equality_expressions)); +} + +Result> KeyValuePartitioning::Partition( + const std::shared_ptr& batch) const { + FieldVector by_fields; + ArrayVector by_columns; + + std::shared_ptr rest = batch; + for (const auto& partition_field : schema_->fields()) { + ARROW_ASSIGN_OR_RAISE( + auto match, FieldRef(partition_field->name()).FindOneOrNone(*rest->schema())) + + if (match) { + by_fields.push_back(partition_field); + by_columns.push_back(rest->column(match[0])); + ARROW_ASSIGN_OR_RAISE(rest, rest->RemoveColumn(match[0])); + } + } + + if (by_fields.empty()) { + // no fields to group by; return the whole batch + return std::vector{{batch, scalar(true)}}; + } + + ARROW_ASSIGN_OR_RAISE(auto by, + StructArray::Make(std::move(by_columns), std::move(by_fields))); + ARROW_ASSIGN_OR_RAISE(auto groupings_and_values, MakeGroupings(*by)); + auto groupings = + checked_pointer_cast(groupings_and_values->GetFieldByName("groupings")); + auto unique_rows = groupings_and_values->GetFieldByName("values"); + + ARROW_ASSIGN_OR_RAISE(auto grouped_batches, ApplyGroupings(*groupings, rest)); + + std::vector out(grouped_batches.size()); + for (size_t i = 0; i < out.size(); ++i) { + ARROW_ASSIGN_OR_RAISE(auto row, unique_rows->GetScalar(i)); + out[i].partition_expression = ConjunctionFromGroupingRow(row.get()); + out[i].batch = std::move(grouped_batches[i]); + } + return out; +} + Result> KeyValuePartitioning::ConvertKey( const Key& key) const { ARROW_ASSIGN_OR_RAISE(auto match, FieldRef(key.name).FindOneOrNone(*schema_)); @@ -254,7 +300,7 @@ Result DirectoryPartitioning::FormatValues( if (auto illegal_index = NextValid(values, i)) { // XXX maybe we should just ignore keys provided after the first absent one? return Status::Invalid("No partition key for ", schema_->field(i)->name(), - " but subsequent a key was provided subsequently for ", + " but a key was provided subsequently for ", schema_->field(*illegal_index)->name(), "."); } @@ -311,6 +357,8 @@ class KeyValuePartitioningInspectImpl { void InsertRepr(int index, std::string repr) { values_[index].insert(std::move(repr)); } Result> Finish(ArrayVector* dictionaries) { + dictionaries->clear(); + if (options_.max_partition_dictionary_size != 0) { dictionaries->resize(name_to_index_.size()); } @@ -391,253 +439,12 @@ class DirectoryPartitioningFactory : public PartitioningFactory { return std::make_shared(std::move(out_schema), dictionaries_); } - struct MakeWritePlanImpl; - - Result MakeWritePlan(std::shared_ptr schema, - FragmentIterator fragments) override; - - Result MakeWritePlan(std::shared_ptr schema, - FragmentIterator fragments, - std::shared_ptr partition_schema) override; - private: std::vector field_names_; ArrayVector dictionaries_; PartitioningFactoryOptions options_; }; -struct DirectoryPartitioningFactory::MakeWritePlanImpl { - using Indices = std::basic_string; - - MakeWritePlanImpl(DirectoryPartitioningFactory* factory, std::shared_ptr schema, - FragmentVector source_fragments) - : this_(factory), - schema_(std::move(schema)), - source_fragments_(std::move(source_fragments)), - right_hand_sides_(source_fragments_.size(), Indices(num_fields(), -1)) {} - - int num_fields() const { return static_cast(this_->field_names_.size()); } - - // For a KeyValuePartitioning, every partition expression will be an equality - // ComparisonExpression where the left operand is a FieldExpression and the right is a - // ScalarExpression. Comparing Scalars directly is expensive, so first assemble a - // dictionary containing the scalars from the right operands of every partition - // expression. This allows later stages of MakeWritePlan to handle a scalar by its - // dictionary code, which is both more compact to store and cheap to compare. - // - // Scalars are stored such that the dictionary code of a fragment's RHS in the - // partition expression for a given field is given by - // int code = right_hand_sides_[fragment_index][field_index]; - // and the corresponding scalar can be retrieved with - // std::shared_ptr scalar = scalar_dict_.code_to_scalar[code]; - Status DictEncodeRightHandSides() { - if (source_fragments_.empty()) { - return Status::OK(); - } - - for (size_t fragment_i = 0; fragment_i < source_fragments_.size(); ++fragment_i) { - const auto& fragment = source_fragments_[fragment_i]; - - auto insert_representable_into_dict = [this, fragment_i]( - const std::string& name, - const std::shared_ptr& value) { - auto it = std::find(this_->field_names_.begin(), this_->field_names_.end(), name); - if (it == this_->field_names_.end()) { - return Status::OK(); - } - - auto field_i = it - this_->field_names_.begin(); - - int code = scalar_dict_.GetOrInsert(value); - right_hand_sides_[fragment_i][field_i] = code; - - return Status::OK(); - }; - - RETURN_NOT_OK(KeyValuePartitioning::VisitKeys(*fragment->partition_expression(), - insert_representable_into_dict)); - - auto it = std::find(right_hand_sides_[fragment_i].begin(), - right_hand_sides_[fragment_i].end(), -1); - if (it != right_hand_sides_[fragment_i].end()) { - // NB: this is an error when writing DirectoryPartitioning but not - // HivePartitioning (as it will be valid to simply omit segments) - return Status::Invalid( - "fragment ", fragment_i, " had no partition expression for field '", - this_->field_names_.at(it - right_hand_sides_[fragment_i].begin()), "'"); - } - } - - return Status::OK(); - } - - // Infer the Partitioning schema from partition expressions. - // For example if one partition expression is "omega"_ == 13 - // we can infer that the field "omega" has type int32 - Result> InferPartitioningSchema() const { - if (source_fragments_.empty()) { - return Status::Invalid( - "No fragments were provided so the Partitioning schema could not be " - "inferred."); - } - - // NB: under DirectoryPartitioning every fragment has a partition expression for every - // field, so we can infer the schema by looking only at the first fragment. This will - // be more complicated for HivePartitioning. - int fragment_i = 0; - - FieldVector fields(num_fields()); - for (int field_i = 0; field_i < num_fields(); ++field_i) { - const auto& name = this_->field_names_[field_i]; - const auto& type = - scalar_dict_.code_to_scalar[right_hand_sides_[fragment_i][field_i]]->type; - fields[field_i] = field(name, type); - } - - return schema(std::move(fields)); - } - - // reconstitute fragment_i's partition expression for field_i by reading the right - // hand side from the scalar dictionary and constructing an equality - // ComparisonExpression - std::shared_ptr PartitionExpression(size_t fragment_i, int field_i) { - auto left_hand_side = field_ref(this_->field_names_[field_i]); - auto right_hand_side = - scalar(scalar_dict_.code_to_scalar[right_hand_sides_[fragment_i][field_i]]); - return equal(std::move(left_hand_side), std::move(right_hand_side)); - } - - // create a guid by stringifying the number of milliseconds since the epoch - std::string Guid() { - using std::chrono::duration_cast; - using std::chrono::milliseconds; - using std::chrono::steady_clock; - auto milliseconds_since_epoch = - duration_cast(steady_clock::now().time_since_epoch()).count(); - return std::to_string(milliseconds_since_epoch); - } - - Result Finish(std::shared_ptr partitioning_schema = nullptr) && { - WritePlan out; - - RETURN_NOT_OK(DictEncodeRightHandSides()); - - if (partitioning_schema == nullptr) { - ARROW_ASSIGN_OR_RAISE(partitioning_schema, InferPartitioningSchema()); - } - ARROW_ASSIGN_OR_RAISE(out.partitioning, - this_->Finish(std::move(partitioning_schema))); - - // There's no guarantee that all Fragments have the same schema. - ARROW_ASSIGN_OR_RAISE(out.schema, - UnifySchemas({out.partitioning->schema(), schema_})); - - // Lexicographic ordering WRT right_hand_sides_ ensures that source_fragments_ are in - // a depth first visitation order WRT their partition expressions. This makes - // generation of the full directory tree far simpler since a directory's files are - // grouped. - auto permutation = arrow::internal::ArgSort(right_hand_sides_); - arrow::internal::Permute(permutation, &source_fragments_); - arrow::internal::Permute(permutation, &right_hand_sides_); - - // out.paths[parents[i]] is the parent directory of out.paths[i] - std::vector parents; - - // current_right_hand_sides[field_i] is the RHS dictionary code for the current - // partition expression corresponding to field_i - Indices current_right_hand_sides(num_fields(), -1); - - // current_partition_expressions[field_i] is the current partition expression - // corresponding to field_i - ExpressionVector current_partition_expressions(num_fields()); - - // out.paths[current_parents[field_i]] is the current ancestor directory corresponding - // to field_i - Indices current_parents(num_fields() + 1, -1); - - for (size_t fragment_i = 0; fragment_i < source_fragments_.size(); ++fragment_i) { - int field_i = 0; - for (; field_i < num_fields(); ++field_i) { - // these directories have already been created and we're still writing their - // children - if (right_hand_sides_[fragment_i][field_i] != current_right_hand_sides[field_i]) { - break; - } - } - - for (; field_i < num_fields(); ++field_i) { - // push a new directory - current_parents[field_i + 1] = static_cast(parents.size()); - parents.push_back(current_parents[field_i]); - - current_partition_expressions.resize(field_i + 1); - current_partition_expressions[field_i] = PartitionExpression(fragment_i, field_i); - auto partition_expression = and_(current_partition_expressions); - - // format segment for partition_expression - ARROW_ASSIGN_OR_RAISE(auto path, out.partitioning->Format(*partition_expression)); - out.paths.push_back(std::move(path)); - - // store partition_expression for use in the written Dataset - out.fragment_or_partition_expressions.emplace_back( - current_partition_expressions[field_i]); - - current_right_hand_sides[field_i] = right_hand_sides_[fragment_i][field_i]; - } - - // push a fragment (not attempting to give files meaningful names) - std::string basename = Guid() + "_" + std::to_string(fragment_i); - int parent_i = current_parents[field_i]; - parents.push_back(parent_i); - out.paths.push_back(fs::internal::JoinAbstractPath( - std::vector{out.paths[parent_i], std::move(basename)})); - - // store a fragment for writing to disk - out.fragment_or_partition_expressions.emplace_back( - std::move(source_fragments_[fragment_i])); - } - - return out; - } - - DirectoryPartitioningFactory* this_; - std::shared_ptr schema_; - FragmentVector source_fragments_; - - struct { - std::unordered_map, int, Scalar::Hash, Scalar::PtrsEqual> - scalar_to_code; - - ScalarVector code_to_scalar; - - int GetOrInsert(const std::shared_ptr& scalar) { - int new_code = static_cast(code_to_scalar.size()); - - auto it_inserted = scalar_to_code.emplace(scalar, new_code); - if (!it_inserted.second) { - return it_inserted.first->second; - } - - code_to_scalar.push_back(scalar); - return new_code; - } - } scalar_dict_; - std::vector right_hand_sides_; -}; - -Result DirectoryPartitioningFactory::MakeWritePlan( - std::shared_ptr schema, FragmentIterator fragment_it, - std::shared_ptr partition_schema) { - ARROW_ASSIGN_OR_RAISE(auto fragments, fragment_it.ToVector()); - return MakeWritePlanImpl(this, schema, std::move(fragments)).Finish(partition_schema); -} - -Result DirectoryPartitioningFactory::MakeWritePlan( - std::shared_ptr schema, FragmentIterator fragment_it) { - ARROW_ASSIGN_OR_RAISE(auto fragments, fragment_it.ToVector()); - return MakeWritePlanImpl(this, schema, std::move(fragments)).Finish(); -} - std::shared_ptr DirectoryPartitioning::MakeFactory( std::vector field_names, PartitioningFactoryOptions options) { return std::shared_ptr( diff --git a/cpp/src/arrow/dataset/partition.h b/cpp/src/arrow/dataset/partition.h index ea5828ec568c..021f82245e2a 100644 --- a/cpp/src/arrow/dataset/partition.h +++ b/cpp/src/arrow/dataset/partition.h @@ -59,6 +59,15 @@ class ARROW_DS_EXPORT Partitioning { /// \brief The name identifying the kind of partitioning virtual std::string type_name() const = 0; + /// \brief If the input batch shares any fields with this partitioning, + /// produce slices of the batch which satisfy mutually exclusive Expressions. + struct PartitionedBatch { + std::shared_ptr batch; + std::shared_ptr partition_expression; + }; + virtual Result> Partition( + const std::shared_ptr& batch) const = 0; + /// \brief Parse a path into a partition expression virtual Result> Parse(const std::string& path) const = 0; @@ -104,15 +113,6 @@ class ARROW_DS_EXPORT PartitioningFactory { /// (fields may be dropped). virtual Result> Finish( const std::shared_ptr& schema) const = 0; - - // FIXME(bkietz) Make these pure virtual - /// Construct a WritePlan for the provided fragments - virtual Result MakeWritePlan(std::shared_ptr schema, - FragmentIterator fragments, - std::shared_ptr partition_schema); - /// Construct a WritePlan for the provided fragments, inferring schema - virtual Result MakeWritePlan(std::shared_ptr schema, - FragmentIterator fragments); }; /// \brief Subclass for the common case of a partitioning which yields an equality @@ -136,6 +136,9 @@ class ARROW_DS_EXPORT KeyValuePartitioning : public Partitioning { static Status SetDefaultValuesFromKeys(const Expression& expr, RecordBatchProjector* projector); + Result> Partition( + const std::shared_ptr& batch) const override; + Result> Parse(const std::string& path) const override; Result Format(const Expression& expr) const override; @@ -240,6 +243,12 @@ class ARROW_DS_EXPORT FunctionPartitioning : public Partitioning { return Status::NotImplemented("formatting paths from ", type_name(), " Partitioning"); } + Result> Partition( + const std::shared_ptr& batch) const override { + return Status::NotImplemented("partitioning batches from ", type_name(), + " Partitioning"); + } + private: ParseImpl parse_impl_; FormatImpl format_impl_; diff --git a/cpp/src/arrow/dataset/partition_test.cc b/cpp/src/arrow/dataset/partition_test.cc index 785be6a99bcf..27ab00ab0c29 100644 --- a/cpp/src/arrow/dataset/partition_test.cc +++ b/cpp/src/arrow/dataset/partition_test.cc @@ -28,6 +28,7 @@ #include #include "arrow/dataset/file_base.h" +#include "arrow/dataset/scanner_internal.h" #include "arrow/dataset/test_util.h" #include "arrow/filesystem/localfs.h" #include "arrow/filesystem/path_util.h" @@ -36,6 +37,8 @@ #include "arrow/util/io_util.h" namespace arrow { +using internal::checked_pointer_cast; + namespace dataset { using E = TestExpression; @@ -436,6 +439,10 @@ class RangePartitioning : public Partitioning { } Result Format(const Expression&) const override { return ""; } + Result> Partition( + const std::shared_ptr&) const override { + return Status::OK(); + } }; TEST_F(TestPartitioning, Range) { @@ -447,178 +454,6 @@ TEST_F(TestPartitioning, Range) { ("z"_ > 1.5 and "z"_ <= 3.0)); } -class TestPartitioningWritePlan : public ::testing::Test { - protected: - FragmentIterator MakeFragments(const ExpressionVector& partition_expressions) { - fragments_.clear(); - for (const auto& expr : partition_expressions) { - fragments_.emplace_back(new InMemoryFragment(RecordBatchVector{}, expr)); - } - return MakeVectorIterator(fragments_); - } - - std::shared_ptr ExpressionPtr(const Expression& e) { return e.Copy(); } - std::shared_ptr ExpressionPtr(std::shared_ptr e) { return e; } - - template - FragmentIterator MakeFragments(const E&... partition_expressions) { - return MakeFragments(ExpressionVector{ExpressionPtr(partition_expressions)...}); - } - - template - void MakeWritePlan(const E&... partition_expressions) { - auto fragments = MakeFragments(partition_expressions...); - EXPECT_OK_AND_ASSIGN(plan_, - factory_->MakeWritePlan(schema({}), std::move(fragments))); - } - - template - Status MakeWritePlanError(const E&... partition_expressions) { - auto fragments = MakeFragments(partition_expressions...); - return factory_->MakeWritePlan(schema({}), std::move(fragments)).status(); - } - - template - void MakeWritePlanWithSchema(const std::shared_ptr& partition_schema, - const E&... partition_expressions) { - auto fragments = MakeFragments(partition_expressions...); - EXPECT_OK_AND_ASSIGN(plan_, factory_->MakeWritePlan(schema({}), std::move(fragments), - partition_schema)); - } - - template - Status MakeWritePlanWithSchemaError(const std::shared_ptr& partition_schema, - const E&... partition_expressions) { - auto fragments = MakeFragments(partition_expressions...); - return factory_->MakeWritePlan(schema({}), std::move(fragments), partition_schema) - .status(); - } - - struct ExpectedWritePlan { - ExpectedWritePlan() = default; - - ExpectedWritePlan(const WritePlan& actual_plan, const FragmentVector& fragments) { - int i = 0; - for (const auto& op : actual_plan.fragment_or_partition_expressions) { - if (op.kind() == WritePlan::FragmentOrPartitionExpression::FRAGMENT) { - auto fragment = op.fragment(); - auto fragment_index = - static_cast(std::find(fragments.begin(), fragments.end(), fragment) - - fragments.begin()); - auto path = fs::internal::GetAbstractPathParent(actual_plan.paths[i]).first; - dirs_[path].fragments.push_back(fragment_index); - } else { - auto partition_expression = op.partition_expr(); - dirs_[actual_plan.paths[i]].partition_expression = partition_expression; - } - ++i; - } - } - - ExpectedWritePlan Dir(const std::string& path, const Expression& expr, - const std::vector& fragments) && { - dirs_.emplace(path, DirectoryWriteOp{expr.Copy(), fragments}); - return std::move(*this); - } - - struct DirectoryWriteOp { - std::shared_ptr partition_expression; - std::vector fragments; - - bool operator==(const DirectoryWriteOp& other) const { - return partition_expression->Equals(other.partition_expression) && - fragments == other.fragments; - } - - friend void PrintTo(const DirectoryWriteOp& op, std::ostream* os) { - *os << op.partition_expression->ToString(); - - *os << " { "; - for (const auto& fragment : op.fragments) { - *os << fragment << " "; - } - *os << "}\n"; - } - }; - std::map dirs_; - }; - - struct AssertPlanIs : ExpectedWritePlan {}; - - void AssertPlanIs(ExpectedWritePlan expected_plan) { - ExpectedWritePlan actual_plan(plan_, fragments_); - EXPECT_THAT(actual_plan.dirs_, testing::ContainerEq(expected_plan.dirs_)); - } - - FragmentVector fragments_; - std::shared_ptr scan_options_ = ScanOptions::Make(schema({})); - std::shared_ptr factory_; - WritePlan plan_; -}; - -TEST_F(TestPartitioningWritePlan, Empty) { - factory_ = DirectoryPartitioning::MakeFactory({"a", "b"}); - - // no expressions from which to infer the types of fields a, b - EXPECT_RAISES_WITH_MESSAGE_THAT(Invalid, testing::HasSubstr("No fragments"), - MakeWritePlanError()); - - MakeWritePlanWithSchema(schema({field("a", int32()), field("b", utf8())})); - AssertPlanIs({}); - - factory_ = HivePartitioning::MakeFactory(); - EXPECT_RAISES_WITH_MESSAGE_THAT(NotImplemented, testing::HasSubstr("hive"), - MakeWritePlanError()); -} - -TEST_F(TestPartitioningWritePlan, SingleDirectory) { - factory_ = DirectoryPartitioning::MakeFactory({"a"}); - - MakeWritePlan("a"_ == 42, "a"_ == 99, "a"_ == 101); - AssertPlanIs(ExpectedWritePlan() - .Dir("42", "a"_ == 42, {0}) - .Dir("99", "a"_ == 99, {1}) - .Dir("101", "a"_ == 101, {2})); - - MakeWritePlan("a"_ == 42, "a"_ == 99, "a"_ == 99, "a"_ == 101, "a"_ == 99); - AssertPlanIs(ExpectedWritePlan() - .Dir("42", "a"_ == 42, {0}) - .Dir("99", "a"_ == 99, {1, 2, 4}) - .Dir("101", "a"_ == 101, {3})); -} - -TEST_F(TestPartitioningWritePlan, NestedDirectories) { - factory_ = DirectoryPartitioning::MakeFactory({"a", "b"}); - - MakeWritePlan("a"_ == 42 and "b"_ == "hello", "a"_ == 42 and "b"_ == "world", - "a"_ == 99 and "b"_ == "hello", "a"_ == 99 and "b"_ == "world"); - - AssertPlanIs(ExpectedWritePlan() - .Dir("42", "a"_ == 42, {}) - .Dir("42/hello", "b"_ == "hello", {0}) - .Dir("42/world", "b"_ == "world", {1}) - .Dir("99", "a"_ == 99, {}) - .Dir("99/hello", "b"_ == "hello", {2}) - .Dir("99/world", "b"_ == "world", {3})); -} - -TEST_F(TestPartitioningWritePlan, Errors) { - factory_ = DirectoryPartitioning::MakeFactory({"a"}); - EXPECT_RAISES_WITH_MESSAGE_THAT( - Invalid, testing::HasSubstr("no partition expression for field 'a'"), - MakeWritePlanError("a"_ == 42, scalar(true), "a"_ == 101)); - - EXPECT_RAISES_WITH_MESSAGE_THAT( - TypeError, testing::HasSubstr("scalar hello (of type string) is invalid"), - MakeWritePlanError("a"_ == 42, "a"_ == "hello")); - - factory_ = DirectoryPartitioning::MakeFactory({"a", "b"}); - EXPECT_RAISES_WITH_MESSAGE_THAT( - Invalid, testing::HasSubstr("no partition expression for field 'a'"), - MakeWritePlanError("a"_ == 42 and "b"_ == "hello", "a"_ == 99 and "b"_ == "world", - "b"_ == "forever alone")); -} - TEST(TestStripPrefixAndFilename, Basic) { ASSERT_EQ(StripPrefixAndFilename("", ""), ""); ASSERT_EQ(StripPrefixAndFilename("a.csv", ""), ""); @@ -633,7 +468,7 @@ TEST(TestStripPrefixAndFilename, Basic) { EXPECT_THAT(StripPrefixAndFilename(input, "/data"), testing::ElementsAre("year=2019", "year=2019/month=12", "year=2019/month=12/day=01")); -} // namespace dataset +} } // namespace dataset } // namespace arrow diff --git a/cpp/src/arrow/dataset/scanner_internal.h b/cpp/src/arrow/dataset/scanner_internal.h index 5737b1e82acc..94df94470fa1 100644 --- a/cpp/src/arrow/dataset/scanner_internal.h +++ b/cpp/src/arrow/dataset/scanner_internal.h @@ -112,5 +112,42 @@ inline ScanTaskIterator GetScanTaskIterator(FragmentIterator fragments, return MakeFlattenIterator(std::move(maybe_scantask_it)); } +struct FragmentRecordBatchReader : RecordBatchReader { + public: + std::shared_ptr schema() const override { return options_->schema(); } + + Status ReadNext(std::shared_ptr* batch) override { + return iterator_.Next().Value(batch); + } + + static Result> Make( + std::shared_ptr fragment, std::shared_ptr schema, + std::shared_ptr context) { + // ensure schema is cached in fragment + auto options = ScanOptions::Make(std::move(schema)); + RETURN_NOT_OK(KeyValuePartitioning::SetDefaultValuesFromKeys( + *fragment->partition_expression(), &options->projector)); + + auto pool = context->pool; + ARROW_ASSIGN_OR_RAISE(auto scan_tasks, fragment->Scan(options, std::move(context))); + + auto reader = std::make_shared(); + reader->options_ = std::move(options); + reader->fragment_ = std::move(fragment); + reader->iterator_ = ProjectRecordBatch( + MakeFlattenIterator(MakeMaybeMapIterator( + [](std::shared_ptr task) { return task->Execute(); }, + std::move(scan_tasks))), + &reader->options_->projector, pool); + + return reader; + } + + private: + std::shared_ptr options_; + std::shared_ptr fragment_; + RecordBatchIterator iterator_; +}; + } // namespace dataset } // namespace arrow diff --git a/cpp/src/arrow/dataset/test_util.h b/cpp/src/arrow/dataset/test_util.h index 8f22b391d1c4..0a686a19268d 100644 --- a/cpp/src/arrow/dataset/test_util.h +++ b/cpp/src/arrow/dataset/test_util.h @@ -300,8 +300,10 @@ struct MakeFileSystemDatasetMixin { partitions.resize(n_fragments, scalar(true)); } + auto s = schema({}); + MakeFileSystem(infos); - auto format = std::make_shared(); + auto format = std::make_shared(s); std::vector> fragments; for (size_t i = 0; i < n_fragments; i++) { @@ -315,8 +317,8 @@ struct MakeFileSystemDatasetMixin { fragments.push_back(std::move(fragment)); } - ASSERT_OK_AND_ASSIGN(dataset_, FileSystemDataset::Make(schema({}), root_partition, - format, std::move(fragments))); + ASSERT_OK_AND_ASSIGN(dataset_, FileSystemDataset::Make(s, root_partition, format, + std::move(fragments))); } void MakeDatasetFromPathlist(const std::string& pathlist, diff --git a/cpp/src/arrow/dataset/type_fwd.h b/cpp/src/arrow/dataset/type_fwd.h index 44c4bbc9f4ff..089e1dffddea 100644 --- a/cpp/src/arrow/dataset/type_fwd.h +++ b/cpp/src/arrow/dataset/type_fwd.h @@ -83,8 +83,5 @@ using ScanTaskIterator = Iterator>; class RecordBatchProjector; -class WriteTask; -class WritePlan; - } // namespace dataset } // namespace arrow diff --git a/cpp/src/arrow/record_batch.cc b/cpp/src/arrow/record_batch.cc index de56b68adb39..56b27838cbf1 100644 --- a/cpp/src/arrow/record_batch.cc +++ b/cpp/src/arrow/record_batch.cc @@ -173,7 +173,7 @@ Result> RecordBatch::FromStructArray( array->data()->child_data); } -Result> RecordBatch::ToStructArray() const { +Result> RecordBatch::ToStructArray() const { if (num_columns() != 0) { return StructArray::Make(columns(), schema()->fields()); } diff --git a/cpp/src/arrow/record_batch.h b/cpp/src/arrow/record_batch.h index 0d1b1b1fe8b7..63d0bd8ba282 100644 --- a/cpp/src/arrow/record_batch.h +++ b/cpp/src/arrow/record_batch.h @@ -65,7 +65,7 @@ class ARROW_EXPORT RecordBatch { /// Create a struct array whose child arrays are the record batch's columns. /// Note that the record batch's top-level field metadata cannot be reflected /// in the resulting struct array. - Result> ToStructArray() const; + Result> ToStructArray() const; /// \brief Construct record batch from struct array /// @@ -210,7 +210,7 @@ class ARROW_EXPORT RecordBatchReader { } /// \brief Consume entire stream as a vector of record batches - Status ReadAll(std::vector>* batches); + Status ReadAll(RecordBatchVector* batches); /// \brief Read all batches and concatenate as arrow::Table Status ReadAll(std::shared_ptr* table); @@ -221,8 +221,7 @@ class ARROW_EXPORT RecordBatchReader { /// \param[in] schema schema to conform to. Will be inferred from the first /// element if not provided. static Result> Make( - std::vector> batches, - std::shared_ptr schema = NULLPTR); + RecordBatchVector batches, std::shared_ptr schema = NULLPTR); }; } // namespace arrow diff --git a/r/DESCRIPTION b/r/DESCRIPTION index ff455b00e25e..cd11e469ef0a 100644 --- a/r/DESCRIPTION +++ b/r/DESCRIPTION @@ -65,6 +65,7 @@ Collate: 'compute.R' 'config.R' 'csv.R' + 'dataset-write.R' 'dataset.R' 'deprecated.R' 'dictionary.R' diff --git a/r/NAMESPACE b/r/NAMESPACE index 4e5f1309fac1..074859178e8f 100644 --- a/r/NAMESPACE +++ b/r/NAMESPACE @@ -8,6 +8,7 @@ S3method("==",ArrowObject) S3method("[",Array) S3method("[",ChunkedArray) S3method("[",RecordBatch) +S3method("[",Schema) S3method("[",Table) S3method("[[",RecordBatch) S3method("[[",Schema) @@ -239,6 +240,7 @@ export(uint8) export(unify_schemas) export(utf8) export(write_arrow) +export(write_dataset) export(write_feather) export(write_ipc_stream) export(write_parquet) diff --git a/r/R/arrowExports.R b/r/R/arrowExports.R index a98a6cba1f63..c013a710d534 100644 --- a/r/R/arrowExports.R +++ b/r/R/arrowExports.R @@ -452,6 +452,10 @@ dataset___ScanTask__get_batches <- function(scan_task){ .Call(`_arrow_dataset___ScanTask__get_batches` , scan_task) } +dataset___Dataset__Write <- function(ds, schema, format, filesystem, path, partitioning){ + invisible(.Call(`_arrow_dataset___Dataset__Write` , ds, schema, format, filesystem, path, partitioning)) +} + shared_ptr_is_null <- function(xp){ .Call(`_arrow_shared_ptr_is_null` , xp) } diff --git a/r/R/dataset-write.R b/r/R/dataset-write.R new file mode 100644 index 000000000000..8baca1e52bb1 --- /dev/null +++ b/r/R/dataset-write.R @@ -0,0 +1,82 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +#' Write a dataset +#' +#' This function allows you to write a dataset. By writing to more efficient +#' binary storage formats, and by specifying relevant partitioning, you can +#' make it much faster to read and query. +#' +#' @param dataset [Dataset] or `arrow_dplyr_query`. If a `arrow_dplyr_query`, +#' note that `select()` or `filter()` queries are not currently supported. +#' @param path string path to a directory to write to (directory will be +#' created if it does not exist) +#' @param format file format to write the dataset to. Currently only "feather" +#' (aka "ipc") is supported. +#' @param partitioning `Partitioning` or a character vector of columns to +#' use as partition keys (to be written as path segments). Default is to +#' use the current `group_by()` columns. +#' @param hive_style logical: write partition segments as Hive-style +#' (`key1=value1/key2=value2/file.ext`) or as just bare values. Default is `TRUE`. +#' @param ... additional arguments, passed to `dataset$write()` +#' @return The input `dataset`, invisibly +#' @export +write_dataset <- function(dataset, + path, + format = dataset$format$type, + partitioning = dplyr::group_vars(dataset), + hive_style = TRUE, + ...) { + if (inherits(dataset, "arrow_dplyr_query")) { + force(partitioning) # get the group_vars before we drop the object + # TODO: Write a filtered/projected dataset + if (!isTRUE(dataset$filtered_rows)) { + stop("Writing a filtered dataset is not yet supported", call. = FALSE) + } + if (!identical(dataset$selected_columns, set_names(names(dataset$.data)))) { + # TODO: actually, we can do this? + stop("TODO", call. = FALSE) + } + dataset <- dataset$.data + } + if (!inherits(dataset, "Dataset")) { + stop("'dataset' must be a Dataset", call. = FALSE) + # TODO: This does not exist yet (in the R bindings at least) + # dataset <- InMemoryDataset$create(dataset) + } + + if (!inherits(format, "FileFormat")) { + format <- FileFormat$create(format, ...) + } + if (!inherits(format, "IpcFileFormat")) { + stop( + "Unsupported format; datasets currently can only be written to IPC/Feather format", + call. = FALSE + ) + } + + if (!inherits(partitioning, "Partitioning")) { + # TODO: tidyselect? + partition_schema <- dataset$schema[partitioning] + if (isTRUE(hive_style)) { + partitioning <- HivePartitioning$create(partition_schema) + } else { + partitioning <- DirectoryPartitioning$create(partition_schema) + } + } + dataset$write(path, format = format, partitioning = partitioning, ...) +} diff --git a/r/R/dataset.R b/r/R/dataset.R index eb229d7abc77..a50e29724f11 100644 --- a/r/R/dataset.R +++ b/r/R/dataset.R @@ -133,6 +133,9 @@ open_dataset <- function(sources, #' may also replace the dataset's schema by using `ds$schema <- new_schema`. #' This method currently supports only adding, removing, or reordering #' fields in the schema: you cannot alter or cast the field types. +#' - `$write(path, filesystem, schema, format, partitioning)`: writes the +#' dataset to `path` in the `format` file format, partitioned by `partitioning`, +#' and invisibly returns `self`. See [write_dataset()]. #' #' `FileSystemDataset` has the following methods: #' - `$files`: Active binding, returns the files of the `FileSystemDataset` @@ -159,7 +162,20 @@ Dataset <- R6Class("Dataset", inherit = ArrowObject, # Start a new scan of the data # @return A [ScannerBuilder] NewScan = function() unique_ptr(ScannerBuilder, dataset___Dataset__NewScan(self)), - ToString = function() self$schema$ToString() + ToString = function() self$schema$ToString(), + write = function(path, filesystem = NULL, schema = self$schema, format, partitioning) { + if (!inherits(filesystem, "FileSystem")) { + if (grepl("://", path)) { + fs_from_uri <- FileSystem$from_uri(path) + filesystem <- fs_from_uri$fs + path <- fs_from_uri$path + } else { + filesystem <- LocalFileSystem$create() + } + } + dataset___Dataset__Write(self, schema, format, filesystem, path, partitioning) + invisible(self) + } ), active = list( schema = function(schema) { diff --git a/r/R/schema.R b/r/R/schema.R index 963e5f45a41d..ddbf30fe93b2 100644 --- a/r/R/schema.R +++ b/r/R/schema.R @@ -151,6 +151,30 @@ length.Schema <- function(x) x$num_fields } } +#' @export +`[.Schema` <- function(x, i, ...) { + if (is.logical(i)) { + i <- rep_len(i, length(x)) # For R recycling behavior + i <- which(i) + } + if (is.numeric(i)) { + if (all(i < 0)) { + # in R, negative i means "everything but i" + i <- setdiff(seq_len(length(x)), -1 * i) + } + } + fields <- map(i, ~x[[.]]) + invalid <- map_lgl(fields, is.null) + if (any(invalid)) { + stop( + "Invalid field name", ifelse(sum(invalid) > 1, "s: ", ": "), + oxford_paste(i[invalid]), + call. = FALSE + ) + } + shared_ptr(Schema, schema_(fields)) +} + #' @export `$.Schema` <- function(x, name, ...) { assert_that(is.string(name)) diff --git a/r/man/Dataset.Rd b/r/man/Dataset.Rd index 686611ca67d8..1f8ce96f4564 100644 --- a/r/man/Dataset.Rd +++ b/r/man/Dataset.Rd @@ -60,6 +60,9 @@ A \code{Dataset} has the following methods: may also replace the dataset's schema by using \code{ds$schema <- new_schema}. This method currently supports only adding, removing, or reordering fields in the schema: you cannot alter or cast the field types. +\item \verb{$write(path, filesystem, schema, format, partitioning)}: writes the +dataset to \code{path} in the \code{format} file format, partitioned by \code{partitioning}, +and invisibly returns \code{self}. See \code{\link[=write_dataset]{write_dataset()}}. } \code{FileSystemDataset} has the following methods: diff --git a/r/man/write_dataset.Rd b/r/man/write_dataset.Rd new file mode 100644 index 000000000000..54bd3539f3c0 --- /dev/null +++ b/r/man/write_dataset.Rd @@ -0,0 +1,42 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/dataset-write.R +\name{write_dataset} +\alias{write_dataset} +\title{Write a dataset} +\usage{ +write_dataset( + dataset, + path, + format = dataset$format$type, + partitioning = dplyr::group_vars(dataset), + hive_style = TRUE, + ... +) +} +\arguments{ +\item{dataset}{\link{Dataset} or \code{arrow_dplyr_query}. If a \code{arrow_dplyr_query}, +note that \code{select()} or \code{filter()} queries are not currently supported.} + +\item{path}{string path to a directory to write to (directory will be +created if it does not exist)} + +\item{format}{file format to write the dataset to. Currently only "feather" +(aka "ipc") is supported.} + +\item{partitioning}{\code{Partitioning} or a character vector of columns to +use as partition keys (to be written as path segments). Default is to +use the current \code{group_by()} columns.} + +\item{hive_style}{logical: write partition segments as Hive-style +(\code{key1=value1/key2=value2/file.ext}) or as just bare values. Default is \code{TRUE}.} + +\item{...}{additional arguments, passed to \code{dataset$write()}} +} +\value{ +The input \code{dataset}, invisibly +} +\description{ +This function allows you to write a dataset. By writing to more efficient +binary storage formats, and by specifying relevant partitioning, you can +make it much faster to read and query. +} diff --git a/r/src/arrowExports.cpp b/r/src/arrowExports.cpp index 9d0058bb9a35..4a80ed08c9fb 100644 --- a/r/src/arrowExports.cpp +++ b/r/src/arrowExports.cpp @@ -1774,6 +1774,27 @@ RcppExport SEXP _arrow_dataset___ScanTask__get_batches(SEXP scan_task_sexp){ } #endif +// dataset.cpp +#if defined(ARROW_R_WITH_ARROW) +void dataset___Dataset__Write(const std::shared_ptr& ds, const std::shared_ptr& schema, const std::shared_ptr& format, const std::shared_ptr& filesystem, std::string path, const std::shared_ptr& partitioning); +RcppExport SEXP _arrow_dataset___Dataset__Write(SEXP ds_sexp, SEXP schema_sexp, SEXP format_sexp, SEXP filesystem_sexp, SEXP path_sexp, SEXP partitioning_sexp){ +BEGIN_RCPP + Rcpp::traits::input_parameter&>::type ds(ds_sexp); + Rcpp::traits::input_parameter&>::type schema(schema_sexp); + Rcpp::traits::input_parameter&>::type format(format_sexp); + Rcpp::traits::input_parameter&>::type filesystem(filesystem_sexp); + Rcpp::traits::input_parameter::type path(path_sexp); + Rcpp::traits::input_parameter&>::type partitioning(partitioning_sexp); + dataset___Dataset__Write(ds, schema, format, filesystem, path, partitioning); + return R_NilValue; +END_RCPP +} +#else +RcppExport SEXP _arrow_dataset___Dataset__Write(SEXP ds_sexp, SEXP schema_sexp, SEXP format_sexp, SEXP filesystem_sexp, SEXP path_sexp, SEXP partitioning_sexp){ + Rf_error("Cannot call dataset___Dataset__Write(). Please use arrow::install_arrow() to install required runtime libraries. "); +} +#endif + // datatype.cpp #if defined(ARROW_R_WITH_ARROW) bool shared_ptr_is_null(SEXP xp); @@ -6023,6 +6044,7 @@ static const R_CallMethodDef CallEntries[] = { { "_arrow_dataset___Scanner__ToTable", (DL_FUNC) &_arrow_dataset___Scanner__ToTable, 1}, { "_arrow_dataset___Scanner__Scan", (DL_FUNC) &_arrow_dataset___Scanner__Scan, 1}, { "_arrow_dataset___ScanTask__get_batches", (DL_FUNC) &_arrow_dataset___ScanTask__get_batches, 1}, + { "_arrow_dataset___Dataset__Write", (DL_FUNC) &_arrow_dataset___Dataset__Write, 6}, { "_arrow_shared_ptr_is_null", (DL_FUNC) &_arrow_shared_ptr_is_null, 1}, { "_arrow_unique_ptr_is_null", (DL_FUNC) &_arrow_unique_ptr_is_null, 1}, { "_arrow_Int8__initialize", (DL_FUNC) &_arrow_Int8__initialize, 0}, diff --git a/r/src/dataset.cpp b/r/src/dataset.cpp index ade66efd1488..aac809aa68ab 100644 --- a/r/src/dataset.cpp +++ b/r/src/dataset.cpp @@ -289,4 +289,19 @@ std::vector> dataset___ScanTask__get_batches return out; } +// [[arrow::export]] +void dataset___Dataset__Write(const std::shared_ptr& ds, + const std::shared_ptr& schema, + const std::shared_ptr& format, + const std::shared_ptr& filesystem, + std::string path, + const std::shared_ptr& partitioning) { + auto frags = ds->GetFragments(); + auto ctx = std::make_shared(); + ctx->use_threads = true; + StopIfNotOk(ds::FileSystemDataset::Write(schema, format, filesystem, path, partitioning, + ctx, std::move(frags))); + return; +} + #endif diff --git a/r/tests/testthat/test-dataset.R b/r/tests/testthat/test-dataset.R index e78dfd3e2b2e..1e93d41ed144 100644 --- a/r/tests/testthat/test-dataset.R +++ b/r/tests/testthat/test-dataset.R @@ -626,3 +626,104 @@ test_that("Assembling multiple DatasetFactories with DatasetFactory", { expect_scan_result(ds, schm) }) + +test_that("Writing a dataset: CSV->IPC", { + skip_on_os("windows") # https://issues.apache.org/jira/browse/ARROW-9651 + ds <- open_dataset(csv_dir, partitioning = "part", format = "csv") + dst_dir <- make_temp_dir() + write_dataset(ds, dst_dir, format = "feather", partitioning = "int") + expect_true(dir.exists(dst_dir)) + expect_identical(dir(dst_dir), sort(paste("int", c(1:10, 101:110), sep = "="))) + + new_ds <- open_dataset(dst_dir, format = "feather") + + expect_equivalent( + new_ds %>% + select(string = chr, integer = int) %>% + filter(integer > 6 & integer < 11) %>% + collect() %>% + summarize(mean = mean(integer)), + df1 %>% + select(string = chr, integer = int) %>% + filter(integer > 6) %>% + summarize(mean = mean(integer)) + ) +}) + +test_that("Writing a dataset: Parquet->IPC", { + skip_on_os("windows") # https://issues.apache.org/jira/browse/ARROW-9651 + ds <- open_dataset(hive_dir) + dst_dir <- make_temp_dir() + write_dataset(ds, dst_dir, format = "feather", partitioning = "int") + expect_true(dir.exists(dst_dir)) + expect_identical(dir(dst_dir), sort(paste("int", c(1:10, 101:110), sep = "="))) + + new_ds <- open_dataset(dst_dir, format = "feather") + + expect_equivalent( + new_ds %>% + select(string = chr, integer = int, group) %>% + filter(integer > 6 & group == 1) %>% + collect() %>% + summarize(mean = mean(integer)), + df1 %>% + select(string = chr, integer = int) %>% + filter(integer > 6) %>% + summarize(mean = mean(integer)) + ) +}) + +test_that("Dataset writing: dplyr methods", { + skip_on_os("windows") # https://issues.apache.org/jira/browse/ARROW-9651 + ds <- open_dataset(hive_dir) + dst_dir <- tempfile() + # Specify partition vars by group_by + ds %>% group_by(int) %>% write_dataset(dst_dir, format = "feather") + expect_true(dir.exists(dst_dir)) + expect_identical(dir(dst_dir), sort(paste("int", c(1:10, 101:110), sep = "="))) + + # select to specify schema + skip("TODO: select to specify schema") + ds %>% group_by(int) %>% select(lgl, chr) %>% write_dataset(dst_dir, format = "feather") + new_ds <- open_dataset(dst_dir, format = "feather") + + expect_equivalent( + collect(new_ds), + rbind(df1[c("lgl", "chr", "int")], df2[c("lgl", "chr", "int")]) + ) +}) + +test_that("Dataset writing: non-hive", { + skip_on_os("windows") # https://issues.apache.org/jira/browse/ARROW-9651 + ds <- open_dataset(hive_dir) + dst_dir <- tempfile() + write_dataset(ds, dst_dir, format = "feather", partitioning = "int", hive_style = FALSE) + expect_true(dir.exists(dst_dir)) + expect_identical(dir(dst_dir), sort(as.character(c(1:10, 101:110)))) +}) + +test_that("Dataset writing: no partitioning", { + skip_on_os("windows") # https://issues.apache.org/jira/browse/ARROW-9651 + ds <- open_dataset(hive_dir) + dst_dir <- tempfile() + write_dataset(ds, dst_dir, format = "feather", partitioning = NULL) + expect_true(dir.exists(dst_dir)) + expect_true(length(dir(dst_dir)) > 1) +}) + +test_that("Dataset writing: unsupported features/input validation", { + expect_error(write_dataset(4), "'dataset' must be a Dataset") + + ds <- open_dataset(hive_dir) + + expect_error(write_dataset(ds, format = "csv"), "Unsupported format") + expect_error( + filter(ds, int == 4) %>% write_dataset(ds), + "Writing a filtered dataset is not yet supported" + ) + + expect_error( + write_dataset(ds, partitioning = c("int", "NOTACOLUMN"), format = "ipc"), + 'Invalid field name: "NOTACOLUMN"' + ) +}) diff --git a/r/tests/testthat/test-schema.R b/r/tests/testthat/test-schema.R index 6671828ecc3c..23b08da54573 100644 --- a/r/tests/testthat/test-schema.R +++ b/r/tests/testthat/test-schema.R @@ -45,9 +45,27 @@ test_that("Schema $GetFieldByName", { expect_null(schm$GetFieldByName("f")) # TODO: schema(b = double(), b = string())$GetFieldByName("b") # also returns NULL and probably should error bc duplicated names +}) +test_that("Schema extract (returns Field)", { + schm <- schema(b = double(), c = string()) expect_equal(schm$b, field("b", double())) expect_equal(schm[["b"]], field("b", double())) + expect_equal(schm[[1]], field("b", double())) + + expect_null(schm[["ZZZ"]]) + expect_error(schm[[42]]) # Should have better error message +}) + +test_that("Schema slicing", { + schm <- schema(b = double(), c = string(), d = int8()) + expect_equal(schm[2:3], schema(c = string(), d = int8())) + expect_equal(schm[-1], schema(c = string(), d = int8())) + expect_equal(schm[c("d", "c")], schema(d = int8(), c = string())) + expect_equal(schm[c(FALSE, TRUE, TRUE)], schema(c = string(), d = int8())) + expect_error(schm[c("c", "ZZZ")], 'Invalid field name: "ZZZ"') + expect_error(schm[c("XXX", "c", "ZZZ")], 'Invalid field names: "XXX" and "ZZZ"') + }) test_that("reading schema from Buffer", { From 81d3f2657b17436d6d5a6af9aaf6f36c3f5e4ac9 Mon Sep 17 00:00:00 2001 From: Tobias Mayer Date: Thu, 6 Aug 2020 15:06:02 +0900 Subject: [PATCH 003/460] ARROW-9412: [C++] Add non-bundled dependencies to INTERFACE_LINK_LIBRARIES of static libarrow This adds support for using the `arrow_static` CMake target with system static libraries. This adds the following CMake options: * `ARROW_DEPENDENCY_USE_SHARED` (`ON` by default): The default value of `ARROW_*_USE_SHARED` except `ARROW_THRIFT_USE_SHARED` on Windows. * `ARROW_BZ2_USE_SHARED`: Use shared library of bzip2. * `ARROW_LZ4_USE_SHARED`: Use shared library of LZ4. * `ARROW_THRIFT_USE_SHARED` (`OFF` by default on Windows): Use shared library of libthrift. This is `OFF` by default on Windows because it seems that Thrift's DLL support isn't well for not. * `ARROW_UTF8PROC_USE_SHARED`: Use shared library of utf8proc. * `ARROW_SNAPPY_USE_SHARED`: Use shared library of Snappy. * `ARROW_ZSTD_USE_SHARED`: Use shared library of ZStandard. Note that users must specify `ARROW_*_USE_SHARED=OFF` explicitly when they want to use static library. This is a backward incompatible change for some cases. For example, users can use shared library or static library without specifying any options before this change. Because old `FindLz4.cmake` find both shared library and static library. Note that upstream `SnappyConfig.cmake` and `zstdConfig.cmake` are never used with this change. We always use our `Find*.cmake` for them to process `ARROW_*_USE_SHARED`. Note that the `ZSTD_SOURCE` CMake option is renamed to `zstd_SOURCE`. Some dependencies don't support static linking with `ARROW_*_USE_SHARED=OFF` yes. For example, gRPC may ignore it. If `gRPCConfig.cmake` is installed, our `FindgRPCAlt.cmake` isn't used. It means that `ARROW_GRPC_USE_SHARED=OFF` is ignored. We can work on this as further tasks. RE2 is only required by Gandiva but it's searched by `find_package(Arrow)` not `find_package(Gandiva)`. It should be fixed by a follow-up task. Closes #7842 from tobim/ARROW-9412-unbundled-static-dependencies Lead-authored-by: Tobias Mayer Co-authored-by: Sutou Kouhei Signed-off-by: Sutou Kouhei --- .github/workflows/cpp.yml | 9 +- ci/docker/ubuntu-14.04-cpp.dockerfile | 2 +- ci/scripts/PKGBUILD | 3 + ci/scripts/cpp_build.sh | 2 +- cpp/CMakeLists.txt | 32 ++++++- cpp/cmake_modules/BuildUtils.cmake | 21 ++-- cpp/cmake_modules/DefineOptions.cmake | 46 +++++++-- cpp/cmake_modules/FindBrotli.cmake | 23 ++--- cpp/cmake_modules/FindGLOG.cmake | 13 ++- cpp/cmake_modules/FindGPerf.cmake | 4 +- cpp/cmake_modules/FindGTest.cmake | 2 +- cpp/cmake_modules/FindLz4.cmake | 59 ++++++++---- cpp/cmake_modules/FindORC.cmake | 8 +- cpp/cmake_modules/FindRE2.cmake | 11 ++- .../{FindSnappyAlt.cmake => FindSnappy.cmake} | 32 +++++-- cpp/cmake_modules/FindThrift.cmake | 53 ++++++++-- cpp/cmake_modules/FindgRPCAlt.cmake | 37 +++---- cpp/cmake_modules/FindgflagsAlt.cmake | 6 +- cpp/cmake_modules/Findutf8proc.cmake | 41 ++++---- .../{FindZSTD.cmake => Findzstd.cmake} | 33 ++++--- cpp/cmake_modules/ThirdpartyToolchain.cmake | 96 ++++++++++--------- cpp/examples/minimal_build/README.md | 39 ++++---- cpp/examples/minimal_build/docker-compose.yml | 51 ++++++++++ .../{Dockerfile => minimal.dockerfile} | 4 +- cpp/examples/minimal_build/run_static.sh | 10 +- .../system_dependency.dockerfile | 42 ++++++++ cpp/src/arrow/ArrowConfig.cmake.in | 26 +++++ cpp/src/arrow/gpu/CMakeLists.txt | 10 +- cpp/src/gandiva/jni/CMakeLists.txt | 16 ++-- cpp/src/jni/orc/CMakeLists.txt | 5 +- dev/tasks/cpp-examples/github.linux.yml | 46 +++++++++ .../debian.ubuntu-xenial/libarrow-dev.install | 5 + .../libgandiva-dev.install | 1 + .../apache-arrow/debian/libarrow-dev.install | 5 + .../debian/libgandiva-dev.install | 1 + .../apache-arrow/yum/arrow.spec.in | 22 ++++- dev/tasks/python-wheels/win-build.bat | 4 +- dev/tasks/tasks.yml | 21 ++++ python/manylinux1/build_arrow.sh | 3 +- python/manylinux201x/build_arrow.sh | 3 +- 40 files changed, 619 insertions(+), 228 deletions(-) rename cpp/cmake_modules/{FindSnappyAlt.cmake => FindSnappy.cmake} (51%) rename cpp/cmake_modules/{FindZSTD.cmake => Findzstd.cmake} (69%) create mode 100644 cpp/examples/minimal_build/docker-compose.yml rename cpp/examples/minimal_build/{Dockerfile => minimal.dockerfile} (93%) create mode 100644 cpp/examples/minimal_build/system_dependency.dockerfile create mode 100644 dev/tasks/cpp-examples/github.linux.yml diff --git a/.github/workflows/cpp.yml b/.github/workflows/cpp.yml index 716824ec9435..07cd4a91914c 100644 --- a/.github/workflows/cpp.yml +++ b/.github/workflows/cpp.yml @@ -174,15 +174,10 @@ jobs: uses: actions/checkout@v2 with: fetch-depth: 0 - - name: Build Docker image + - name: Run run: | cd cpp/examples/minimal_build - docker build -t arrow_cpp_minimal . - - name: Compile and run example - run: | - ARROW_ROOT=$PWD - cd cpp/examples/minimal_build - docker run -v $PWD:/io -v $ARROW_ROOT:/arrow arrow_cpp_minimal /io/run.sh + docker-compose run --rm minimal macos: name: AMD64 MacOS 10.15 C++ diff --git a/ci/docker/ubuntu-14.04-cpp.dockerfile b/ci/docker/ubuntu-14.04-cpp.dockerfile index 116973e6c804..6335dcc46e7b 100644 --- a/ci/docker/ubuntu-14.04-cpp.dockerfile +++ b/ci/docker/ubuntu-14.04-cpp.dockerfile @@ -91,4 +91,4 @@ ENV ARROW_BUILD_TESTS=ON \ RE2_SOURCE=BUNDLED \ Thrift_SOURCE=BUNDLED \ utf8proc_SOURCE=BUNDLED \ - ZSTD_SOURCE=BUNDLED + zstd_SOURCE=BUNDLED diff --git a/ci/scripts/PKGBUILD b/ci/scripts/PKGBUILD index a813b2357f5d..df5f5860445c 100644 --- a/ci/scripts/PKGBUILD +++ b/ci/scripts/PKGBUILD @@ -90,15 +90,18 @@ build() { -DARROW_HDFS=OFF \ -DARROW_JEMALLOC=OFF \ -DARROW_JSON=ON \ + -DARROW_LZ4_USE_SHARED=OFF \ -DARROW_MIMALLOC=ON \ -DARROW_PACKAGE_PREFIX="${MINGW_PREFIX}" \ -DARROW_PARQUET=ON \ + -DARROW_SNAPPY_USE_SHARED=OFF \ -DARROW_USE_GLOG=OFF \ -DARROW_WITH_LZ4=ON \ -DARROW_WITH_SNAPPY=ON \ -DARROW_WITH_UTF8PROC=OFF \ -DARROW_WITH_ZLIB=ON \ -DARROW_WITH_ZSTD=ON \ + -DARROW_ZSTD_USE_SHARED=OFF \ -DARROW_CXXFLAGS="${CPPFLAGS}" \ -DCMAKE_BUILD_TYPE="release" \ -DCMAKE_INSTALL_PREFIX=${MINGW_PREFIX} \ diff --git a/ci/scripts/cpp_build.sh b/ci/scripts/cpp_build.sh index 2de6254aacc3..aebc02ed2cfd 100755 --- a/ci/scripts/cpp_build.sh +++ b/ci/scripts/cpp_build.sh @@ -125,7 +125,7 @@ cmake -G "${CMAKE_GENERATOR:-Ninja}" \ -DSnappy_SOURCE=${Snappy_SOURCE:-} \ -DThrift_SOURCE=${Thrift_SOURCE:-} \ -Dutf8proc_SOURCE=${utf8proc_SOURCE:-} \ - -DZSTD_SOURCE=${ZSTD_SOURCE:-} \ + -Dzstd_SOURCE=${zstd_SOURCE:-} \ ${CMAKE_ARGS} \ ${source_dir} diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 55d5ed9c258f..721ed22811f2 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -100,7 +100,7 @@ set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_CURRENT_SOURCE_DIR}/cmake_mo include(CMakePackageConfigHelpers) include(CMakeParseArguments) include(ExternalProject) -include(FindPkgConfig) +include(FindPackageHandleStandardArgs) include(GNUInstallDirs) @@ -650,6 +650,7 @@ if(ARROW_USE_OPENSSL) set(ARROW_OPENSSL_LIBS OpenSSL::Crypto OpenSSL::SSL) list(APPEND ARROW_LINK_LIBS ${ARROW_OPENSSL_LIBS}) list(APPEND ARROW_STATIC_LINK_LIBS ${ARROW_OPENSSL_LIBS}) + list(APPEND ARROW_STATIC_INSTALL_INTERFACE_LIBS ${ARROW_OPENSSL_LIBS}) endif() if(ARROW_WITH_BROTLI) @@ -657,36 +658,61 @@ if(ARROW_WITH_BROTLI) set(ARROW_BROTLI_LIBS Brotli::brotlienc Brotli::brotlidec Brotli::brotlicommon) list(APPEND ARROW_LINK_LIBS ${ARROW_BROTLI_LIBS}) list(APPEND ARROW_STATIC_LINK_LIBS ${ARROW_BROTLI_LIBS}) + if(Brotli_SOURCE STREQUAL "SYSTEM") + list(APPEND ARROW_STATIC_INSTALL_INTERFACE_LIBS ${ARROW_BROTLI_LIBS}) + endif() endif() if(ARROW_WITH_BZ2) list(APPEND ARROW_STATIC_LINK_LIBS BZip2::BZip2) + if(BZip2_SOURCE STREQUAL "SYSTEM") + list(APPEND ARROW_STATIC_INSTALL_INTERFACE_LIBS BZip2::BZip2) + endif() endif() if(ARROW_WITH_LZ4) list(APPEND ARROW_STATIC_LINK_LIBS LZ4::lz4) + if(Lz4_SOURCE STREQUAL "SYSTEM") + list(APPEND ARROW_STATIC_INSTALL_INTERFACE_LIBS LZ4::lz4) + endif() endif() if(ARROW_WITH_SNAPPY) list(APPEND ARROW_STATIC_LINK_LIBS Snappy::snappy) + if(Snappy_SOURCE STREQUAL "SYSTEM") + list(APPEND ARROW_STATIC_INSTALL_INTERFACE_LIBS Snappy::snappy) + endif() endif() if(ARROW_WITH_ZLIB) list(APPEND ARROW_STATIC_LINK_LIBS ZLIB::ZLIB) + if(ZLIB_SOURCE STREQUAL "SYSTEM") + list(APPEND ARROW_STATIC_INSTALL_INTERFACE_LIBS ZLIB::ZLIB) + endif() endif() if(ARROW_WITH_ZSTD) list(APPEND ARROW_STATIC_LINK_LIBS ${ARROW_ZSTD_LIBZSTD}) + if(zstd_SOURCE STREQUAL "SYSTEM") + list(APPEND ARROW_STATIC_INSTALL_INTERFACE_LIBS ${ARROW_ZSTD_LIBZSTD}) + endif() endif() if(ARROW_ORC) list(APPEND ARROW_LINK_LIBS orc::liborc ${ARROW_PROTOBUF_LIBPROTOBUF}) list(APPEND ARROW_STATIC_LINK_LIBS orc::liborc ${ARROW_PROTOBUF_LIBPROTOBUF}) + if(ORC_SOURCE STREQUAL "SYSTEM") + list(APPEND ARROW_STATIC_INSTALL_INTERFACE_LIBS orc::liborc + ${ARROW_PROTOBUF_LIBPROTOBUF}) + endif() endif() if(ARROW_USE_GLOG) list(APPEND ARROW_LINK_LIBS glog::glog) list(APPEND ARROW_STATIC_LINK_LIBS glog::glog) + if(GLOG_SOURCE STREQUAL "SYSTEM") + list(APPEND ARROW_STATIC_INSTALL_INTERFACE_LIBS glog::glog) + endif() add_definitions("-DARROW_USE_GLOG") endif() @@ -697,6 +723,9 @@ endif() if(ARROW_WITH_UTF8PROC) list(APPEND ARROW_LINK_LIBS utf8proc::utf8proc) list(APPEND ARROW_STATIC_LINK_LIBS utf8proc::utf8proc) + if(utf8proc_SOURCE STREQUAL "SYSTEM") + list(APPEND ARROW_STATIC_INSTALL_INTERFACE_LIBS utf8proc::utf8proc) + endif() endif() add_custom_target(arrow_dependencies) @@ -825,6 +854,7 @@ endif() list(APPEND ARROW_LINK_LIBS ${ARROW_SYSTEM_LINK_LIBS}) list(APPEND ARROW_STATIC_LINK_LIBS ${ARROW_SYSTEM_LINK_LIBS}) +list(APPEND ARROW_STATIC_INSTALL_INTERFACE_LIBS ${ARROW_SYSTEM_LINK_LIBS}) # # Subdirectories diff --git a/cpp/cmake_modules/BuildUtils.cmake b/cpp/cmake_modules/BuildUtils.cmake index eb10ebeeb1c7..f92966f78ebe 100644 --- a/cpp/cmake_modules/BuildUtils.cmake +++ b/cpp/cmake_modules/BuildUtils.cmake @@ -18,7 +18,7 @@ # Common path suffixes to be searched by find_library or find_path. # Windows artifacts may be found under "/Library", so # search there as well. -set(LIB_PATH_SUFFIXES +set(ARROW_LIBRARY_PATH_SUFFIXES "${CMAKE_LIBRARY_ARCHITECTURE}" "lib/${CMAKE_LIBRARY_ARCHITECTURE}" "lib64" @@ -28,7 +28,7 @@ set(LIB_PATH_SUFFIXES "Library" "Library/lib" "Library/bin") -set(INCLUDE_PATH_SUFFIXES "include" "Library" "Library/include") +set(ARROW_INCLUDE_PATH_SUFFIXES "include" "Library" "Library/include") function(ADD_THIRDPARTY_LIB LIB_NAME) set(options) @@ -213,9 +213,14 @@ endfunction() # \arg OUTPUTS list to append built targets to function(ADD_ARROW_LIB LIB_NAME) - set(options BUILD_SHARED BUILD_STATIC) - set(one_value_args CMAKE_PACKAGE_NAME PKG_CONFIG_NAME SHARED_LINK_FLAGS - PRECOMPILED_HEADER_LIB) + set(options) + set(one_value_args + BUILD_SHARED + BUILD_STATIC + CMAKE_PACKAGE_NAME + PKG_CONFIG_NAME + SHARED_LINK_FLAGS + PRECOMPILED_HEADER_LIB) set(multi_value_args SOURCES PRECOMPILED_HEADERS @@ -243,12 +248,12 @@ function(ADD_ARROW_LIB LIB_NAME) endif() # Allow overriding ARROW_BUILD_SHARED and ARROW_BUILD_STATIC - if(ARG_BUILD_SHARED) + if(DEFINED ARG_BUILD_SHARED) set(BUILD_SHARED ${ARG_BUILD_SHARED}) else() set(BUILD_SHARED ${ARROW_BUILD_SHARED}) endif() - if(ARG_BUILD_STATIC) + if(DEFINED ARG_BUILD_STATIC) set(BUILD_STATIC ${ARG_BUILD_STATIC}) else() set(BUILD_STATIC ${ARROW_BUILD_STATIC}) @@ -334,7 +339,7 @@ function(ADD_ARROW_LIB LIB_NAME) endif() # On iOS, specifying -undefined conflicts with enabling bitcode - if(APPLE AND NOT IOS AND NOT DEFINED $ENV{EMSCRIPTEN}) + if(APPLE AND NOT IOS AND NOT DEFINED ENV{EMSCRIPTEN}) # On OS X, you can avoid linking at library load time and instead # expecting that the symbols have been loaded separately. This happens # with libpython* where there can be conflicts between system Python and diff --git a/cpp/cmake_modules/DefineOptions.cmake b/cpp/cmake_modules/DefineOptions.cmake index da2c6d551915..555099fb4d6e 100644 --- a/cpp/cmake_modules/DefineOptions.cmake +++ b/cpp/cmake_modules/DefineOptions.cmake @@ -284,29 +284,59 @@ if("${CMAKE_SOURCE_DIR}" STREQUAL "${CMAKE_CURRENT_SOURCE_DIR}") define_option(ARROW_VERBOSE_THIRDPARTY_BUILD "Show output from ExternalProjects rather than just logging to files" OFF) - define_option(ARROW_BOOST_USE_SHARED "Rely on boost shared libraries where relevant" ON) + define_option(ARROW_DEPENDENCY_USE_SHARED "Link to shared libraries" ON) + + define_option(ARROW_BOOST_USE_SHARED "Rely on boost shared libraries where relevant" + ${ARROW_DEPENDENCY_USE_SHARED}) define_option(ARROW_BROTLI_USE_SHARED "Rely on Brotli shared libraries where relevant" - ON) + ${ARROW_DEPENDENCY_USE_SHARED}) + + define_option(ARROW_BZ2_USE_SHARED "Rely on Bz2 shared libraries where relevant" + ${ARROW_DEPENDENCY_USE_SHARED}) define_option(ARROW_GFLAGS_USE_SHARED "Rely on GFlags shared libraries where relevant" - ON) + ${ARROW_DEPENDENCY_USE_SHARED}) + + define_option(ARROW_GRPC_USE_SHARED "Rely on gRPC shared libraries where relevant" + ${ARROW_DEPENDENCY_USE_SHARED}) - define_option(ARROW_GRPC_USE_SHARED "Rely on gRPC shared libraries where relevant" ON) + define_option(ARROW_LZ4_USE_SHARED "Rely on lz4 shared libraries where relevant" + ${ARROW_DEPENDENCY_USE_SHARED}) define_option(ARROW_PROTOBUF_USE_SHARED - "Rely on Protocol Buffers shared libraries where relevant" ON) + "Rely on Protocol Buffers shared libraries where relevant" + ${ARROW_DEPENDENCY_USE_SHARED}) + + if(WIN32) + # It seems that Thrift doesn't support DLL well yet. + # MSYS2, conda-forge and vcpkg don't build shared library. + set(ARROW_THRIFT_USE_SHARED_DEFAULT OFF) + else() + set(ARROW_THRIFT_USE_SHARED_DEFAULT ${ARROW_DEPENDENCY_USE_SHARED}) + endif() + define_option(ARROW_THRIFT_USE_SHARED "Rely on thrift shared libraries where relevant" + ${ARROW_THRIFT_USE_SHARED_DEFAULT}) define_option(ARROW_UTF8PROC_USE_SHARED - "Rely on utf8proc shared libraries where relevant" ON) + "Rely on utf8proc shared libraries where relevant" + ${ARROW_DEPENDENCY_USE_SHARED}) - define_option(ARROW_ZSTD_USE_SHARED "Rely on zstd shared libraries where relevant" ON) + define_option(ARROW_SNAPPY_USE_SHARED "Rely on snappy shared libraries where relevant" + ${ARROW_DEPENDENCY_USE_SHARED}) - define_option(ARROW_WITH_BACKTRACE "Build with backtrace support" ON) + define_option(ARROW_UTF8PROC_USE_SHARED + "Rely on utf8proc shared libraries where relevant" + ${ARROW_DEPENDENCY_USE_SHARED}) + + define_option(ARROW_ZSTD_USE_SHARED "Rely on zstd shared libraries where relevant" + ${ARROW_DEPENDENCY_USE_SHARED}) define_option(ARROW_USE_GLOG "Build libraries with glog support for pluggable logging" OFF) + define_option(ARROW_WITH_BACKTRACE "Build with backtrace support" ON) + define_option(ARROW_WITH_BROTLI "Build with Brotli compression" OFF) define_option(ARROW_WITH_BZ2 "Build with BZ2 compression" OFF) define_option(ARROW_WITH_LZ4 "Build with lz4 compression" OFF) diff --git a/cpp/cmake_modules/FindBrotli.cmake b/cpp/cmake_modules/FindBrotli.cmake index dc74c5ce58b2..b46a0f1a0cf4 100644 --- a/cpp/cmake_modules/FindBrotli.cmake +++ b/cpp/cmake_modules/FindBrotli.cmake @@ -51,24 +51,25 @@ if(BROTLI_ROOT) find_library(BROTLI_COMMON_LIBRARY NAMES ${BROTLI_COMMON_LIB_NAMES} PATHS ${BROTLI_ROOT} - PATH_SUFFIXES ${LIB_PATH_SUFFIXES} + PATH_SUFFIXES ${ARROW_LIBRARY_PATH_SUFFIXES} NO_DEFAULT_PATH) find_library(BROTLI_ENC_LIBRARY NAMES ${BROTLI_ENC_LIB_NAMES} PATHS ${BROTLI_ROOT} - PATH_SUFFIXES ${LIB_PATH_SUFFIXES} + PATH_SUFFIXES ${ARROW_LIBRARY_PATH_SUFFIXES} NO_DEFAULT_PATH) find_library(BROTLI_DEC_LIBRARY NAMES ${BROTLI_DEC_LIB_NAMES} PATHS ${BROTLI_ROOT} - PATH_SUFFIXES ${LIB_PATH_SUFFIXES} + PATH_SUFFIXES ${ARROW_LIBRARY_PATH_SUFFIXES} NO_DEFAULT_PATH) find_path(BROTLI_INCLUDE_DIR NAMES brotli/decode.h PATHS ${BROTLI_ROOT} - PATH_SUFFIXES ${INCLUDE_PATH_SUFFIXES} + PATH_SUFFIXES ${ARROW_INCLUDE_PATH_SUFFIXES} NO_DEFAULT_PATH) else() + find_package(PkgConfig QUIET) pkg_check_modules(BROTLI_PC libbrotlicommon libbrotlienc libbrotlidec) if(BROTLI_PC_FOUND) set(BROTLI_INCLUDE_DIR "${BROTLI_PC_libbrotlicommon_INCLUDEDIR}") @@ -81,31 +82,31 @@ else() find_library(BROTLI_COMMON_LIBRARY NAMES ${BROTLI_COMMON_LIB_NAMES} PATHS ${BROTLI_PC_LIBRARY_DIRS} - PATH_SUFFIXES ${LIB_PATH_SUFFIXES} + PATH_SUFFIXES ${ARROW_LIBRARY_PATH_SUFFIXES} NO_DEFAULT_PATH) find_library(BROTLI_ENC_LIBRARY NAMES ${BROTLI_ENC_LIB_NAMES} PATHS ${BROTLI_PC_LIBRARY_DIRS} - PATH_SUFFIXES ${LIB_PATH_SUFFIXES} + PATH_SUFFIXES ${ARROW_LIBRARY_PATH_SUFFIXES} NO_DEFAULT_PATH) find_library(BROTLI_DEC_LIBRARY NAMES ${BROTLI_DEC_LIB_NAMES} PATHS ${BROTLI_PC_LIBRARY_DIRS} - PATH_SUFFIXES ${LIB_PATH_SUFFIXES} + PATH_SUFFIXES ${ARROW_LIBRARY_PATH_SUFFIXES} NO_DEFAULT_PATH) else() find_library(BROTLI_COMMON_LIBRARY NAMES ${BROTLI_COMMON_LIB_NAMES} - PATH_SUFFIXES ${LIB_PATH_SUFFIXES}) + PATH_SUFFIXES ${ARROW_LIBRARY_PATH_SUFFIXES}) find_library(BROTLI_ENC_LIBRARY NAMES ${BROTLI_ENC_LIB_NAMES} - PATH_SUFFIXES ${LIB_PATH_SUFFIXES}) + PATH_SUFFIXES ${ARROW_LIBRARY_PATH_SUFFIXES}) find_library(BROTLI_DEC_LIBRARY NAMES ${BROTLI_DEC_LIB_NAMES} - PATH_SUFFIXES ${LIB_PATH_SUFFIXES}) + PATH_SUFFIXES ${ARROW_LIBRARY_PATH_SUFFIXES}) find_path(BROTLI_INCLUDE_DIR NAMES brotli/decode.h - PATH_SUFFIXES ${INCLUDE_PATH_SUFFIXES}) + PATH_SUFFIXES ${ARROW_INCLUDE_PATH_SUFFIXES}) endif() endif() diff --git a/cpp/cmake_modules/FindGLOG.cmake b/cpp/cmake_modules/FindGLOG.cmake index e64a22f1f385..81c3f2ec57e2 100644 --- a/cpp/cmake_modules/FindGLOG.cmake +++ b/cpp/cmake_modules/FindGLOG.cmake @@ -17,28 +17,31 @@ # # find_package(GLOG) +find_package(PkgConfig QUIET) pkg_check_modules(GLOG_PC libglog) if(GLOG_PC_FOUND) set(GLOG_INCLUDE_DIR "${GLOG_PC_INCLUDEDIR}") list(APPEND GLOG_PC_LIBRARY_DIRS "${GLOG_PC_LIBDIR}") find_library(GLOG_LIB glog PATHS ${GLOG_PC_LIBRARY_DIRS} - PATH_SUFFIXES ${LIB_PATH_SUFFIXES} + PATH_SUFFIXES ${ARROW_LIBRARY_PATH_SUFFIXES} NO_DEFAULT_PATH) elseif(GLOG_ROOT) find_library(GLOG_LIB NAMES glog PATHS ${GLOG_ROOT} - PATH_SUFFIXES ${LIB_PATH_SUFFIXES} + PATH_SUFFIXES ${ARROW_LIBRARY_PATH_SUFFIXES} NO_DEFAULT_PATH) find_path(GLOG_INCLUDE_DIR NAMES glog/logging.h PATHS ${GLOG_ROOT} NO_DEFAULT_PATH - PATH_SUFFIXES ${INCLUDE_PATH_SUFFIXES}) + PATH_SUFFIXES ${ARROW_INCLUDE_PATH_SUFFIXES}) else() - find_library(GLOG_LIB NAMES glog PATH_SUFFIXES ${LIB_PATH_SUFFIXES}) - find_path(GLOG_INCLUDE_DIR NAMES glog/logging.h PATH_SUFFIXES ${INCLUDE_PATH_SUFFIXES}) + find_library(GLOG_LIB NAMES glog PATH_SUFFIXES ${ARROW_LIBRARY_PATH_SUFFIXES}) + find_path(GLOG_INCLUDE_DIR + NAMES glog/logging.h + PATH_SUFFIXES ${ARROW_INCLUDE_PATH_SUFFIXES}) endif() find_package_handle_standard_args(GLOG REQUIRED_VARS GLOG_INCLUDE_DIR GLOG_LIB) diff --git a/cpp/cmake_modules/FindGPerf.cmake b/cpp/cmake_modules/FindGPerf.cmake index 96e507fe75b9..7b9b15a50c71 100644 --- a/cpp/cmake_modules/FindGPerf.cmake +++ b/cpp/cmake_modules/FindGPerf.cmake @@ -35,7 +35,7 @@ SET(GPERF_LIB_SEARCH $ENV{NATIVE_TOOLCHAIN}/gperftools-$ENV{GPERFTOOLS_VERSION}/ FIND_LIBRARY(TCMALLOC_LIB_PATH NAMES libtcmalloc.a PATHS ${GPERF_LIB_SEARCH} - PATH_SUFFIXES ${LIB_PATH_SUFFIXES} + PATH_SUFFIXES ${ARROW_LIBRARY_PATH_SUFFIXES} NO_DEFAULT_PATH ) @@ -52,7 +52,7 @@ ENDIF (TCMALLOC_LIB_PATH AND GOOGLE_PERFTOOLS_INCLUDE_DIR) FIND_LIBRARY(PROFILER_LIB_PATH NAMES libprofiler.a PATHS ${GPERF_LIB_SEARCH} - PATH_SUFFIXES ${LIB_PATH_SUFFIXES} + PATH_SUFFIXES ${ARROW_LIBRARY_PATH_SUFFIXES} ) IF (PROFILER_LIB_PATH AND GOOGLE_PERFTOOLS_INCLUDE_DIR) diff --git a/cpp/cmake_modules/FindGTest.cmake b/cpp/cmake_modules/FindGTest.cmake index 1a22929c9246..4be2cf570468 100644 --- a/cpp/cmake_modules/FindGTest.cmake +++ b/cpp/cmake_modules/FindGTest.cmake @@ -194,7 +194,7 @@ find_path(GTEST_INCLUDE_DIR gtest/gtest.h HINTS $ENV{GTEST_ROOT}/include ${GTEST_ROOT}/include - PATH_SUFFIXES ${LIB_PATH_SUFFIXES} + PATH_SUFFIXES ${ARROW_LIBRARY_PATH_SUFFIXES} ) mark_as_advanced(GTEST_INCLUDE_DIR) diff --git a/cpp/cmake_modules/FindLz4.cmake b/cpp/cmake_modules/FindLz4.cmake index 8410916432ac..dbc6c5f8d3be 100644 --- a/cpp/cmake_modules/FindLz4.cmake +++ b/cpp/cmake_modules/FindLz4.cmake @@ -15,45 +15,62 @@ # specific language governing permissions and limitations # under the License. -if(MSVC AND NOT DEFINED LZ4_MSVC_STATIC_LIB_SUFFIX) - set(LZ4_MSVC_STATIC_LIB_SUFFIX "_static") +if(MSVC AND NOT DEFINED LZ4_MSVC_LIB_PREFIX) + set(LZ4_MSVC_LIB_PREFIX "lib") endif() +set(LZ4_LIB_NAME_BASE "${LZ4_MSVC_LIB_PREFIX}lz4") -set(LZ4_STATIC_LIB_SUFFIX "${LZ4_MSVC_STATIC_LIB_SUFFIX}${CMAKE_STATIC_LIBRARY_SUFFIX}") - -set(LZ4_STATIC_LIB_NAME ${CMAKE_STATIC_LIBRARY_PREFIX}lz4${LZ4_STATIC_LIB_SUFFIX}) +if(ARROW_LZ4_USE_SHARED) + set(LZ4_LIB_NAMES) + if(CMAKE_IMPORT_LIBRARY_SUFFIX) + list( + APPEND + LZ4_LIB_NAMES + "${CMAKE_IMPORT_LIBRARY_PREFIX}${LZ4_LIB_NAME_BASE}${CMAKE_IMPORT_LIBRARY_SUFFIX}" + ) + endif() + list( + APPEND + LZ4_LIB_NAMES + "${CMAKE_SHARED_LIBRARY_PREFIX}${LZ4_LIB_NAME_BASE}${CMAKE_SHARED_LIBRARY_SUFFIX}") +else() + if(MSVC AND NOT DEFINED LZ4_MSVC_STATIC_LIB_SUFFIX) + set(LZ4_MSVC_STATIC_LIB_SUFFIX "_static") + endif() + set(LZ4_STATIC_LIB_SUFFIX "${LZ4_MSVC_STATIC_LIB_SUFFIX}${CMAKE_STATIC_LIBRARY_SUFFIX}") + set(LZ4_LIB_NAMES + "${CMAKE_STATIC_LIBRARY_PREFIX}${LZ4_LIB_NAME_BASE}${LZ4_STATIC_LIB_SUFFIX}") +endif() if(LZ4_ROOT) - find_library( - LZ4_LIB - NAMES lz4 ${LZ4_STATIC_LIB_NAME} lib${LZ4_STATIC_LIB_NAME} - "${CMAKE_SHARED_LIBRARY_PREFIX}lz4_static${CMAKE_SHARED_LIBRARY_SUFFIX}" - PATHS ${LZ4_ROOT} - PATH_SUFFIXES ${LIB_PATH_SUFFIXES} - NO_DEFAULT_PATH) + find_library(LZ4_LIB + NAMES ${LZ4_LIB_NAMES} + PATHS ${LZ4_ROOT} + PATH_SUFFIXES ${ARROW_LIBRARY_PATH_SUFFIXES} + NO_DEFAULT_PATH) find_path(LZ4_INCLUDE_DIR NAMES lz4.h PATHS ${LZ4_ROOT} NO_DEFAULT_PATH - PATH_SUFFIXES ${INCLUDE_PATH_SUFFIXES}) + PATH_SUFFIXES ${ARROW_INCLUDE_PATH_SUFFIXES}) else() + find_package(PkgConfig QUIET) pkg_check_modules(LZ4_PC liblz4) if(LZ4_PC_FOUND) set(LZ4_INCLUDE_DIR "${LZ4_PC_INCLUDEDIR}") list(APPEND LZ4_PC_LIBRARY_DIRS "${LZ4_PC_LIBDIR}") - find_library(LZ4_LIB lz4 + find_library(LZ4_LIB + NAMES ${LZ4_LIB_NAMES} PATHS ${LZ4_PC_LIBRARY_DIRS} NO_DEFAULT_PATH - PATH_SUFFIXES ${LIB_PATH_SUFFIXES}) + PATH_SUFFIXES ${ARROW_LIBRARY_PATH_SUFFIXES}) else() - find_library( - LZ4_LIB - NAMES lz4 ${LZ4_STATIC_LIB_NAME} lib${LZ4_STATIC_LIB_NAME} - "${CMAKE_SHARED_LIBRARY_PREFIX}lz4_static${CMAKE_SHARED_LIBRARY_SUFFIX}" - PATH_SUFFIXES ${LIB_PATH_SUFFIXES}) - find_path(LZ4_INCLUDE_DIR NAMES lz4.h PATH_SUFFIXES ${INCLUDE_PATH_SUFFIXES}) + find_library(LZ4_LIB + NAMES ${LZ4_LIB_NAMES} + PATH_SUFFIXES ${ARROW_LIBRARY_PATH_SUFFIXES}) + find_path(LZ4_INCLUDE_DIR NAMES lz4.h PATH_SUFFIXES ${ARROW_INCLUDE_PATH_SUFFIXES}) endif() endif() diff --git a/cpp/cmake_modules/FindORC.cmake b/cpp/cmake_modules/FindORC.cmake index 1cda71fee0ac..1be149c93b2a 100644 --- a/cpp/cmake_modules/FindORC.cmake +++ b/cpp/cmake_modules/FindORC.cmake @@ -26,17 +26,17 @@ if(ORC_ROOT) NAMES orc PATHS ${ORC_ROOT} NO_DEFAULT_PATH - PATH_SUFFIXES ${LIB_PATH_SUFFIXES}) + PATH_SUFFIXES ${ARROW_LIBRARY_PATH_SUFFIXES}) find_path(ORC_INCLUDE_DIR NAMES orc/orc-config.hh PATHS ${ORC_ROOT} NO_DEFAULT_PATH - PATH_SUFFIXES ${INCLUDE_PATH_SUFFIXES}) + PATH_SUFFIXES ${ARROW_INCLUDE_PATH_SUFFIXES}) else() - find_library(ORC_STATIC_LIB NAMES orc PATH_SUFFIXES ${LIB_PATH_SUFFIXES}) + find_library(ORC_STATIC_LIB NAMES orc PATH_SUFFIXES ${ARROW_LIBRARY_PATH_SUFFIXES}) find_path(ORC_INCLUDE_DIR NAMES orc/orc-config.hh - PATH_SUFFIXES ${INCLUDE_PATH_SUFFIXES}) + PATH_SUFFIXES ${ARROW_INCLUDE_PATH_SUFFIXES}) endif() if(ORC_STATIC_LIB AND ORC_INCLUDE_DIR) diff --git a/cpp/cmake_modules/FindRE2.cmake b/cpp/cmake_modules/FindRE2.cmake index b8d76cef1947..4562e3ae89bb 100644 --- a/cpp/cmake_modules/FindRE2.cmake +++ b/cpp/cmake_modules/FindRE2.cmake @@ -15,6 +15,7 @@ # specific language governing permissions and limitations # under the License. +find_package(PkgConfig QUIET) pkg_check_modules(RE2_PC re2) if(RE2_PC_FOUND) set(RE2_INCLUDE_DIR "${RE2_PC_INCLUDEDIR}") @@ -22,7 +23,7 @@ if(RE2_PC_FOUND) list(APPEND RE2_PC_LIBRARY_DIRS "${RE2_PC_LIBDIR}") find_library(RE2_LIB re2 PATHS ${RE2_PC_LIBRARY_DIRS} - PATH_SUFFIXES ${LIB_PATH_SUFFIXES} + PATH_SUFFIXES ${ARROW_LIBRARY_PATH_SUFFIXES} NO_DEFAULT_PATH) # On Fedora, the reported prefix is wrong. As users likely run into this, @@ -41,13 +42,13 @@ elseif(RE2_ROOT) "${CMAKE_STATIC_LIBRARY_PREFIX}re2${RE2_MSVC_STATIC_LIB_SUFFIX}${CMAKE_STATIC_LIBRARY_SUFFIX}" "${CMAKE_SHARED_LIBRARY_PREFIX}re2${CMAKE_SHARED_LIBRARY_SUFFIX}" PATHS ${RE2_ROOT} - PATH_SUFFIXES ${LIB_PATH_SUFFIXES} + PATH_SUFFIXES ${ARROW_LIBRARY_PATH_SUFFIXES} NO_DEFAULT_PATH) find_path(RE2_INCLUDE_DIR NAMES re2/re2.h PATHS ${RE2_ROOT} NO_DEFAULT_PATH - PATH_SUFFIXES ${INCLUDE_PATH_SUFFIXES}) + PATH_SUFFIXES ${ARROW_INCLUDE_PATH_SUFFIXES}) else() find_library( RE2_LIB @@ -55,8 +56,8 @@ else() re2_static re2 "${CMAKE_STATIC_LIBRARY_PREFIX}re2${RE2_MSVC_STATIC_LIB_SUFFIX}${CMAKE_STATIC_LIBRARY_SUFFIX}" "${CMAKE_SHARED_LIBRARY_PREFIX}re2${CMAKE_SHARED_LIBRARY_SUFFIX}" - PATH_SUFFIXES ${LIB_PATH_SUFFIXES}) - find_path(RE2_INCLUDE_DIR NAMES re2/re2.h PATH_SUFFIXES ${INCLUDE_PATH_SUFFIXES}) + PATH_SUFFIXES ${ARROW_LIBRARY_PATH_SUFFIXES}) + find_path(RE2_INCLUDE_DIR NAMES re2/re2.h PATH_SUFFIXES ${ARROW_INCLUDE_PATH_SUFFIXES}) endif() find_package_handle_standard_args(RE2 REQUIRED_VARS RE2_LIB RE2_INCLUDE_DIR) diff --git a/cpp/cmake_modules/FindSnappyAlt.cmake b/cpp/cmake_modules/FindSnappy.cmake similarity index 51% rename from cpp/cmake_modules/FindSnappyAlt.cmake rename to cpp/cmake_modules/FindSnappy.cmake index 019c8794f18a..7be7fb264d10 100644 --- a/cpp/cmake_modules/FindSnappyAlt.cmake +++ b/cpp/cmake_modules/FindSnappy.cmake @@ -15,25 +15,43 @@ # specific language governing permissions and limitations # under the License. +if(ARROW_SNAPPY_USE_SHARED) + set(SNAPPY_LIB_NAMES) + if(CMAKE_IMPORT_LIBRARY_SUFFIX) + list(APPEND SNAPPY_LIB_NAMES + "${CMAKE_IMPORT_LIBRARY_PREFIX}snappy${CMAKE_IMPORT_LIBRARY_SUFFIX}") + endif() + list(APPEND SNAPPY_LIB_NAMES + "${CMAKE_SHARED_LIBRARY_PREFIX}snappy${CMAKE_SHARED_LIBRARY_SUFFIX}") +else() + if(MSVC AND DEFINED ENV{CONDA_PREFIX}) + # Conda package changes the output name. + # https://github.com/conda-forge/snappy-feedstock/blob/master/recipe/windows-static-lib-name.patch + set(SNAPPY_LIB_NAMES "${CMAKE_STATIC_LIBRARY_PREFIX}snappy_static${CMAKE_STATIC_LIBRARY_SUFFIX}") + endif() + set(SNAPPY_LIB_NAMES "${CMAKE_STATIC_LIBRARY_PREFIX}snappy${CMAKE_STATIC_LIBRARY_SUFFIX}") +endif() + if(Snappy_ROOT) find_library(Snappy_LIB - NAMES snappy + NAMES ${SNAPPY_LIB_NAMES} PATHS ${Snappy_ROOT} - PATH_SUFFIXES ${LIB_PATH_SUFFIXES} + PATH_SUFFIXES ${ARROW_LIBRARY_PATH_SUFFIXES} NO_DEFAULT_PATH) find_path(Snappy_INCLUDE_DIR NAMES snappy.h PATHS ${Snappy_ROOT} NO_DEFAULT_PATH - PATH_SUFFIXES ${INCLUDE_PATH_SUFFIXES}) + PATH_SUFFIXES ${ARROW_INCLUDE_PATH_SUFFIXES}) else() - find_library(Snappy_LIB NAMES snappy) - find_path(Snappy_INCLUDE_DIR NAMES snappy.h PATH_SUFFIXES ${INCLUDE_PATH_SUFFIXES}) + find_library(Snappy_LIB NAMES ${SNAPPY_LIB_NAMES}) + find_path(Snappy_INCLUDE_DIR NAMES snappy.h PATH_SUFFIXES ${ARROW_INCLUDE_PATH_SUFFIXES}) endif() -find_package_handle_standard_args(SnappyAlt REQUIRED_VARS Snappy_LIB Snappy_INCLUDE_DIR) +find_package_handle_standard_args(Snappy REQUIRED_VARS Snappy_LIB Snappy_INCLUDE_DIR) -if(SnappyAlt_FOUND) +# CMake 3.2 does uppercase the FOUND variable +if(Snappy_FOUND OR SNAPPY_FOUND) add_library(Snappy::snappy UNKNOWN IMPORTED) set_target_properties(Snappy::snappy PROPERTIES IMPORTED_LOCATION "${Snappy_LIB}" diff --git a/cpp/cmake_modules/FindThrift.cmake b/cpp/cmake_modules/FindThrift.cmake index bb3eb5608592..d266f02e29b8 100644 --- a/cpp/cmake_modules/FindThrift.cmake +++ b/cpp/cmake_modules/FindThrift.cmake @@ -24,7 +24,7 @@ # This module defines # THRIFT_VERSION, version string of ant if found # THRIFT_INCLUDE_DIR, where to find THRIFT headers -# THRIFT_STATIC_LIB, THRIFT static library +# THRIFT_LIB, THRIFT library # THRIFT_FOUND, If false, do not try to use ant function(EXTRACT_THRIFT_VERSION) @@ -39,12 +39,40 @@ function(EXTRACT_THRIFT_VERSION) endif() endfunction(EXTRACT_THRIFT_VERSION) -if(MSVC AND NOT THRIFT_MSVC_STATIC_LIB_SUFFIX) - set(THRIFT_MSVC_STATIC_LIB_SUFFIX md) +if(MSVC AND NOT DEFINED THRIFT_MSVC_LIB_SUFFIX) + if(NOT ARROW_THRIFT_USE_SHARED) + if(ARROW_USE_STATIC_CRT) + set(THRIFT_MSVC_LIB_SUFFIX "mt") + else() + set(THRIFT_MSVC_LIB_SUFFIX "md") + endif() + endif() +endif() +set(THRIFT_LIB_NAME_BASE "thrift${THRIFT_MSVC_LIB_SUFFIX}") + +if(ARROW_THRIFT_USE_SHARED) + set(THRIFT_LIB_NAMES thrift) + if(CMAKE_IMPORT_LIBRARY_SUFFIX) + list( + APPEND + THRIFT_LIB_NAMES + "${CMAKE_IMPORT_LIBRARY_PREFIX}${THRIFT_LIB_NAME_BASE}${CMAKE_IMPORT_LIBRARY_SUFFIX}" + ) + endif() + list( + APPEND + THRIFT_LIB_NAMES + "${CMAKE_SHARED_LIBRARY_PREFIX}${THRIFT_LIB_NAME_BASE}${CMAKE_SHARED_LIBRARY_SUFFIX}" + ) +else() + set( + THRIFT_LIB_NAMES + "${CMAKE_STATIC_LIBRARY_PREFIX}${THRIFT_LIB_NAME_BASE}${CMAKE_STATIC_LIBRARY_SUFFIX}") endif() if(Thrift_ROOT) - find_library(THRIFT_STATIC_LIB thrift${THRIFT_MSVC_STATIC_LIB_SUFFIX} + find_library(THRIFT_LIB + NAMES ${THRIFT_LIB_NAMES} PATHS ${Thrift_ROOT} PATH_SUFFIXES "lib/${CMAKE_LIBRARY_ARCHITECTURE}" "lib") find_path(THRIFT_INCLUDE_DIR thrift/Thrift.h @@ -55,13 +83,15 @@ if(Thrift_ROOT) else() # THRIFT-4760: The pkgconfig files are currently only installed when using autotools. # Starting with 0.13, they are also installed for the CMake-based installations of Thrift. + find_package(PkgConfig QUIET) pkg_check_modules(THRIFT_PC thrift) if(THRIFT_PC_FOUND) set(THRIFT_INCLUDE_DIR "${THRIFT_PC_INCLUDEDIR}") list(APPEND THRIFT_PC_LIBRARY_DIRS "${THRIFT_PC_LIBDIR}") - find_library(THRIFT_STATIC_LIB thrift${THRIFT_MSVC_STATIC_LIB_SUFFIX} + find_library(THRIFT_LIB + NAMES ${THRIFT_LIB_NAMES} PATHS ${THRIFT_PC_LIBRARY_DIRS} NO_DEFAULT_PATH) find_program(THRIFT_COMPILER thrift @@ -70,7 +100,8 @@ else() PATH_SUFFIXES "bin") set(THRIFT_VERSION ${THRIFT_PC_VERSION}) else() - find_library(THRIFT_STATIC_LIB thrift${THRIFT_MSVC_STATIC_LIB_SUFFIX} + find_library(THRIFT_LIB + NAMES ${THRIFT_LIB_NAMES} PATH_SUFFIXES "lib/${CMAKE_LIBRARY_ARCHITECTURE}" "lib") find_path(THRIFT_INCLUDE_DIR thrift/Thrift.h PATH_SUFFIXES "include") find_program(THRIFT_COMPILER thrift PATH_SUFFIXES "bin") @@ -86,7 +117,7 @@ endif() find_package_handle_standard_args(Thrift REQUIRED_VARS - THRIFT_STATIC_LIB + THRIFT_LIB THRIFT_INCLUDE_DIR VERSION_VAR THRIFT_VERSION @@ -94,9 +125,13 @@ find_package_handle_standard_args(Thrift if(Thrift_FOUND OR THRIFT_FOUND) set(Thrift_FOUND TRUE) - add_library(thrift::thrift STATIC IMPORTED) + if(ARROW_THRIFT_USE_SHARED) + add_library(thrift::thrift SHARED IMPORTED) + else() + add_library(thrift::thrift STATIC IMPORTED) + endif() set_target_properties(thrift::thrift - PROPERTIES IMPORTED_LOCATION "${THRIFT_STATIC_LIB}" + PROPERTIES IMPORTED_LOCATION "${THRIFT_LIB}" INTERFACE_INCLUDE_DIRECTORIES "${THRIFT_INCLUDE_DIR}") if(WIN32 AND NOT MSVC) # We don't need this for Visual C++ because Thrift uses diff --git a/cpp/cmake_modules/FindgRPCAlt.cmake b/cpp/cmake_modules/FindgRPCAlt.cmake index 35b36827f83b..d6ba331aea39 100644 --- a/cpp/cmake_modules/FindgRPCAlt.cmake +++ b/cpp/cmake_modules/FindgRPCAlt.cmake @@ -61,27 +61,27 @@ if(gRPC_ROOT) find_library(GRPC_GPR_LIB NAMES ${GRPC_GPR_LIB_NAMES} PATHS ${gRPC_ROOT} - PATH_SUFFIXES ${LIB_PATH_SUFFIXES} + PATH_SUFFIXES ${ARROW_LIBRARY_PATH_SUFFIXES} NO_DEFAULT_PATH) find_library(GRPC_GRPC_LIB NAMES ${GRPC_GRPC_LIB_NAMES} PATHS ${gRPC_ROOT} - PATH_SUFFIXES ${LIB_PATH_SUFFIXES} + PATH_SUFFIXES ${ARROW_LIBRARY_PATH_SUFFIXES} NO_DEFAULT_PATH) find_library(GRPC_GRPCPP_LIB NAMES ${GRPC_GRPCPP_LIB_NAMES} PATHS ${gRPC_ROOT} - PATH_SUFFIXES ${LIB_PATH_SUFFIXES} + PATH_SUFFIXES ${ARROW_LIBRARY_PATH_SUFFIXES} NO_DEFAULT_PATH) find_library(GRPC_ADDRESS_SORTING_LIB NAMES ${GRPC_ADDRESS_SORTING_LIB_NAMES} PATHS ${gRPC_ROOT} - PATH_SUFFIXES ${LIB_PATH_SUFFIXES} + PATH_SUFFIXES ${ARROW_LIBRARY_PATH_SUFFIXES} NO_DEFAULT_PATH) find_library(GRPC_UPB_LIB NAMES ${GRPC_UPB_LIB_NAMES} PATHS ${gRPC_ROOT} - PATH_SUFFIXES ${LIB_PATH_SUFFIXES} + PATH_SUFFIXES ${ARROW_LIBRARY_PATH_SUFFIXES} NO_DEFAULT_PATH) find_program(GRPC_CPP_PLUGIN grpc_cpp_plugin NO_DEFAULT_PATH PATHS ${gRPC_ROOT} @@ -90,8 +90,9 @@ if(gRPC_ROOT) NAMES grpc/grpc.h PATHS ${gRPC_ROOT} NO_DEFAULT_PATH - PATH_SUFFIXES ${INCLUDE_PATH_SUFFIXES}) + PATH_SUFFIXES ${ARROW_INCLUDE_PATH_SUFFIXES}) else() + find_package(PkgConfig QUIET) pkg_check_modules(GRPC_PC grpc++) if(GRPC_PC_FOUND) set(GRPC_ALT_VERSION "${GRPC_PC_VERSION}") @@ -102,27 +103,27 @@ else() find_library(GRPC_GPR_LIB NAMES ${GRPC_GPR_LIB_NAMES} PATHS ${GRPC_PC_LIBRARY_DIRS} - PATH_SUFFIXES ${LIB_PATH_SUFFIXES} + PATH_SUFFIXES ${ARROW_LIBRARY_PATH_SUFFIXES} NO_DEFAULT_PATH) find_library(GRPC_GRPC_LIB NAMES ${GRPC_GRPC_LIB_NAMES} PATHS ${GRPC_PC_LIBRARY_DIRS} - PATH_SUFFIXES ${LIB_PATH_SUFFIXES} + PATH_SUFFIXES ${ARROW_LIBRARY_PATH_SUFFIXES} NO_DEFAULT_PATH) find_library(GRPC_GRPCPP_LIB NAMES ${GRPC_GRPCPP_LIB_NAMES} PATHS ${GRPC_PC_LIBRARY_DIRS} - PATH_SUFFIXES ${LIB_PATH_SUFFIXES} + PATH_SUFFIXES ${ARROW_LIBRARY_PATH_SUFFIXES} NO_DEFAULT_PATH) find_library(GRPC_ADDRESS_SORTING_LIB NAMES ${GRPC_ADDRESS_SORTING_LIB_NAMES} PATHS ${GRPC_PC_LIBRARY_DIRS} - PATH_SUFFIXES ${LIB_PATH_SUFFIXES} + PATH_SUFFIXES ${ARROW_LIBRARY_PATH_SUFFIXES} NO_DEFAULT_PATH) find_library(GRPC_UPB_LIB NAMES ${GRPC_UPB_LIB_NAMES} PATHS ${GRPC_PC_LIBRARY_DIRS} - PATH_SUFFIXES ${LIB_PATH_SUFFIXES} + PATH_SUFFIXES ${ARROW_LIBRARY_PATH_SUFFIXES} NO_DEFAULT_PATH) find_program(GRPC_CPP_PLUGIN grpc_cpp_plugin HINTS ${GRPC_PC_PREFIX} @@ -131,21 +132,23 @@ else() else() find_library(GRPC_GPR_LIB NAMES ${GRPC_GPR_LIB_NAMES} - PATH_SUFFIXES ${LIB_PATH_SUFFIXES}) + PATH_SUFFIXES ${ARROW_LIBRARY_PATH_SUFFIXES}) find_library(GRPC_GRPC_LIB NAMES ${GRPC_GRPC_LIB_NAMES} - PATH_SUFFIXES ${LIB_PATH_SUFFIXES}) + PATH_SUFFIXES ${ARROW_LIBRARY_PATH_SUFFIXES}) find_library(GRPC_GRPCPP_LIB NAMES ${GRPC_GRPCPP_LIB_NAMES} - PATH_SUFFIXES ${LIB_PATH_SUFFIXES}) + PATH_SUFFIXES ${ARROW_LIBRARY_PATH_SUFFIXES}) find_library(GRPC_ADDRESS_SORTING_LIB NAMES ${GRPC_ADDRESS_SORTING_LIB_NAMES} - PATH_SUFFIXES ${LIB_PATH_SUFFIXES}) + PATH_SUFFIXES ${ARROW_LIBRARY_PATH_SUFFIXES}) find_library(GRPC_UPB_LIB NAMES ${GRPC_UPB_LIB_NAMES} - PATH_SUFFIXES ${LIB_PATH_SUFFIXES}) + PATH_SUFFIXES ${ARROW_LIBRARY_PATH_SUFFIXES}) find_program(GRPC_CPP_PLUGIN grpc_cpp_plugin PATH_SUFFIXES "bin") - find_path(GRPC_INCLUDE_DIR NAMES grpc/grpc.h PATH_SUFFIXES ${INCLUDE_PATH_SUFFIXES}) + find_path(GRPC_INCLUDE_DIR + NAMES grpc/grpc.h + PATH_SUFFIXES ${ARROW_INCLUDE_PATH_SUFFIXES}) endif() endif() diff --git a/cpp/cmake_modules/FindgflagsAlt.cmake b/cpp/cmake_modules/FindgflagsAlt.cmake index 556048c6076f..cf6d2654f73a 100644 --- a/cpp/cmake_modules/FindgflagsAlt.cmake +++ b/cpp/cmake_modules/FindgflagsAlt.cmake @@ -21,18 +21,18 @@ if(gflags_ROOT) find_library(gflags_LIB NAMES gflags PATHS ${gflags_ROOT} - PATH_SUFFIXES ${LIB_PATH_SUFFIXES} + PATH_SUFFIXES ${ARROW_LIBRARY_PATH_SUFFIXES} NO_DEFAULT_PATH) find_path(GFLAGS_INCLUDE_DIR NAMES gflags/gflags.h PATHS ${gflags_ROOT} NO_DEFAULT_PATH - PATH_SUFFIXES ${INCLUDE_PATH_SUFFIXES}) + PATH_SUFFIXES ${ARROW_INCLUDE_PATH_SUFFIXES}) else() find_library(gflags_LIB NAMES gflags) find_path(GFLAGS_INCLUDE_DIR NAMES gflags/gflags.h - PATH_SUFFIXES ${INCLUDE_PATH_SUFFIXES}) + PATH_SUFFIXES ${ARROW_INCLUDE_PATH_SUFFIXES}) endif() find_package_handle_standard_args(gflagsAlt REQUIRED_VARS gflags_LIB GFLAGS_INCLUDE_DIR) diff --git a/cpp/cmake_modules/Findutf8proc.cmake b/cpp/cmake_modules/Findutf8proc.cmake index d3063827add9..e3ae0ca87f1f 100644 --- a/cpp/cmake_modules/Findutf8proc.cmake +++ b/cpp/cmake_modules/Findutf8proc.cmake @@ -16,52 +16,51 @@ # under the License. if(ARROW_UTF8PROC_USE_SHARED) - set(UTF8PROC_LIB_NAMES) + set(utf8proc_LIB_NAMES) if(CMAKE_IMPORT_LIBRARY_SUFFIX) - list(APPEND UTF8PROC_LIB_NAMES + list(APPEND utf8proc_LIB_NAMES "${CMAKE_IMPORT_LIBRARY_PREFIX}utf8proc${CMAKE_IMPORT_LIBRARY_SUFFIX}") endif() - list(APPEND UTF8PROC_LIB_NAMES + list(APPEND utf8proc_LIB_NAMES "${CMAKE_SHARED_LIBRARY_PREFIX}utf8proc${CMAKE_SHARED_LIBRARY_SUFFIX}") else() - if(MSVC AND NOT DEFINED UTF8PROC_MSVC_STATIC_LIB_SUFFIX) - set(UTF8PROC_MSVC_STATIC_LIB_SUFFIX "_static") + if(MSVC AND NOT DEFINED utf8proc_MSVC_STATIC_LIB_SUFFIX) + set(utf8proc_MSVC_STATIC_LIB_SUFFIX "_static") endif() - set(UTF8PROC_STATIC_LIB_SUFFIX - "${UTF8PROC_MSVC_STATIC_LIB_SUFFIX}${CMAKE_STATIC_LIBRARY_SUFFIX}") - set(UTF8PROC_STATIC_LIB_NAME ${CMAKE_STATIC_LIBRARY_PREFIX}utf8proc${UTF8PROC_STATIC_LIB_SUFFIX}) - set(UTF8PROC_LIB_NAMES "${UTF8PROC_STATIC_LIB_NAME}" "lib${UTF8PROC_STATIC_LIB_NAME}") + set(utf8proc_STATIC_LIB_SUFFIX + "${utf8proc_MSVC_STATIC_LIB_SUFFIX}${CMAKE_STATIC_LIBRARY_SUFFIX}") + set(utf8proc_LIB_NAMES "${CMAKE_STATIC_LIBRARY_PREFIX}utf8proc${utf8proc_STATIC_LIB_SUFFIX}") endif() if(utf8proc_ROOT) find_library( - UTF8PROC_LIB - NAMES ${UTF8PROC_LIB_NAMES} + utf8proc_LIB + NAMES ${utf8proc_LIB_NAMES} PATHS ${utf8proc_ROOT} - PATH_SUFFIXES ${LIB_PATH_SUFFIXES} + PATH_SUFFIXES ${ARROW_LIBRARY_PATH_SUFFIXES} NO_DEFAULT_PATH) - find_path(UTF8PROC_INCLUDE_DIR + find_path(utf8proc_INCLUDE_DIR NAMES utf8proc.h PATHS ${utf8proc_ROOT} NO_DEFAULT_PATH - PATH_SUFFIXES ${INCLUDE_PATH_SUFFIXES}) + PATH_SUFFIXES ${ARROW_INCLUDE_PATH_SUFFIXES}) else() find_library( - UTF8PROC_LIB - NAMES ${UTF8PROC_LIB_NAMES} - PATH_SUFFIXES ${LIB_PATH_SUFFIXES}) - find_path(UTF8PROC_INCLUDE_DIR NAMES utf8proc.h PATH_SUFFIXES ${INCLUDE_PATH_SUFFIXES}) + utf8proc_LIB + NAMES ${utf8proc_LIB_NAMES} + PATH_SUFFIXES ${ARROW_LIBRARY_PATH_SUFFIXES}) + find_path(utf8proc_INCLUDE_DIR NAMES utf8proc.h PATH_SUFFIXES ${ARROW_INCLUDE_PATH_SUFFIXES}) endif() -find_package_handle_standard_args(utf8proc REQUIRED_VARS UTF8PROC_LIB UTF8PROC_INCLUDE_DIR) +find_package_handle_standard_args(utf8proc REQUIRED_VARS utf8proc_LIB utf8proc_INCLUDE_DIR) # CMake 3.2 does uppercase the FOUND variable if(UTF8PROC_FOUND OR utf8proc_FOUND) set(utf8proc_FOUND TRUE) add_library(utf8proc::utf8proc UNKNOWN IMPORTED) set_target_properties(utf8proc::utf8proc - PROPERTIES IMPORTED_LOCATION "${UTF8PROC_LIB}" - INTERFACE_INCLUDE_DIRECTORIES "${UTF8PROC_INCLUDE_DIR}") + PROPERTIES IMPORTED_LOCATION "${utf8proc_LIB}" + INTERFACE_INCLUDE_DIRECTORIES "${utf8proc_INCLUDE_DIR}") if(NOT ARROW_UTF8PROC_USE_SHARED) set_target_properties(utf8proc::utf8proc PROPERTIES INTERFACE_COMPILER_DEFINITIONS "UTF8PROC_STATIC") diff --git a/cpp/cmake_modules/FindZSTD.cmake b/cpp/cmake_modules/Findzstd.cmake similarity index 69% rename from cpp/cmake_modules/FindZSTD.cmake rename to cpp/cmake_modules/Findzstd.cmake index 84d21d2b5c73..f7c68134e9d7 100644 --- a/cpp/cmake_modules/FindZSTD.cmake +++ b/cpp/cmake_modules/Findzstd.cmake @@ -15,22 +15,26 @@ # specific language governing permissions and limitations # under the License. +if(MSVC AND NOT DEFINED ZSTD_MSVC_LIB_PREFIX) + set(ZSTD_MSVC_LIB_PREFIX "lib") +endif() +set(ZSTD_LIB_NAME_BASE "${ZSTD_MSVC_LIB_PREFIX}zstd") + if(ARROW_ZSTD_USE_SHARED) set(ZSTD_LIB_NAMES) if(CMAKE_IMPORT_LIBRARY_SUFFIX) list(APPEND ZSTD_LIB_NAMES - "${CMAKE_IMPORT_LIBRARY_PREFIX}zstd${CMAKE_IMPORT_LIBRARY_SUFFIX}") + "${CMAKE_IMPORT_LIBRARY_PREFIX}${ZSTD_LIB_NAME_BASE}${CMAKE_IMPORT_LIBRARY_SUFFIX}") endif() list(APPEND ZSTD_LIB_NAMES - "${CMAKE_SHARED_LIBRARY_PREFIX}zstd${CMAKE_SHARED_LIBRARY_SUFFIX}") + "${CMAKE_SHARED_LIBRARY_PREFIX}${ZSTD_LIB_NAME_BASE}${CMAKE_SHARED_LIBRARY_SUFFIX}") else() if(MSVC AND NOT DEFINED ZSTD_MSVC_STATIC_LIB_SUFFIX) set(ZSTD_MSVC_STATIC_LIB_SUFFIX "_static") endif() set(ZSTD_STATIC_LIB_SUFFIX "${ZSTD_MSVC_STATIC_LIB_SUFFIX}${CMAKE_STATIC_LIBRARY_SUFFIX}") - set(ZSTD_STATIC_LIB_NAME ${CMAKE_STATIC_LIBRARY_PREFIX}zstd${ZSTD_STATIC_LIB_SUFFIX}) - set(ZSTD_LIB_NAMES "${ZSTD_STATIC_LIB_NAME}" "lib${ZSTD_STATIC_LIB_NAME}") + set(ZSTD_LIB_NAMES "${CMAKE_STATIC_LIBRARY_PREFIX}${ZSTD_LIB_NAME_BASE}${ZSTD_STATIC_LIB_SUFFIX}") endif() # First, find via if specified ZTD_ROOT @@ -39,35 +43,38 @@ if(ZSTD_ROOT) find_library(ZSTD_LIB NAMES ${ZSTD_LIB_NAMES} PATHS ${ZSTD_ROOT} - PATH_SUFFIXES ${LIB_PATH_SUFFIXES} + PATH_SUFFIXES ${ARROW_LIBRARY_PATH_SUFFIXES} NO_DEFAULT_PATH) find_path(ZSTD_INCLUDE_DIR NAMES zstd.h PATHS ${ZSTD_ROOT} NO_DEFAULT_PATH - PATH_SUFFIXES ${INCLUDE_PATH_SUFFIXES}) + PATH_SUFFIXES ${ARROW_INCLUDE_PATH_SUFFIXES}) else() # Second, find via pkg_check_modules + find_package(PkgConfig QUIET) pkg_check_modules(ZSTD_PC libzstd) if(ZSTD_PC_FOUND) set(ZSTD_INCLUDE_DIR "${ZSTD_PC_INCLUDEDIR}") list(APPEND ZSTD_PC_LIBRARY_DIRS "${ZSTD_PC_LIBDIR}") - find_library(ZSTD_LIB zstd + find_library(ZSTD_LIB + NAMES ${ZSTD_LIB_NAMES} PATHS ${ZSTD_PC_LIBRARY_DIRS} NO_DEFAULT_PATH - PATH_SUFFIXES ${LIB_PATH_SUFFIXES}) - # Third, check all other CMake paths + PATH_SUFFIXES ${ARROW_LIBRARY_PATH_SUFFIXES}) else() - find_library(ZSTD_LIB NAMES ${ZSTD_LIB_NAMES} PATH_SUFFIXES ${LIB_PATH_SUFFIXES}) - find_path(ZSTD_INCLUDE_DIR NAMES zstd.h PATH_SUFFIXES ${INCLUDE_PATH_SUFFIXES}) + # Third, check all other CMake paths + find_library(ZSTD_LIB NAMES ${ZSTD_LIB_NAMES} PATH_SUFFIXES ${ARROW_LIBRARY_PATH_SUFFIXES}) + find_path(ZSTD_INCLUDE_DIR NAMES zstd.h PATH_SUFFIXES ${ARROW_INCLUDE_PATH_SUFFIXES}) endif() endif() -find_package_handle_standard_args(ZSTD REQUIRED_VARS ZSTD_LIB ZSTD_INCLUDE_DIR) +find_package_handle_standard_args(zstd REQUIRED_VARS ZSTD_LIB ZSTD_INCLUDE_DIR) -if(ZSTD_FOUND) +# CMake 3.2 does uppercase the FOUND variable +if(zstd_FOUND OR ZSTD_FOUND) add_library(zstd::libzstd UNKNOWN IMPORTED) set_target_properties(zstd::libzstd PROPERTIES IMPORTED_LOCATION "${ZSTD_LIB}" diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index 807e2b9206b4..f983d58f9b95 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -29,6 +29,10 @@ add_custom_target(toolchain-tests) # allocators like jemalloc and mimalloc set(ARROW_BUNDLED_STATIC_LIBS) +# Accumulate all system dependencies to provide suitable static link +# parameters to the third party libraries. +set(ARROW_SYSTEM_DEPENDENCIES) + # ---------------------------------------------------------------------- # Toolchain linkage options @@ -66,7 +70,7 @@ set(ARROW_THIRDPARTY_DEPENDENCIES Thrift utf8proc ZLIB - ZSTD) + zstd) # TODO(wesm): External GTest shared libraries are not currently # supported when building with MSVC because of the way that @@ -140,42 +144,68 @@ macro(build_dependency DEPENDENCY_NAME) build_protobuf() elseif("${DEPENDENCY_NAME}" STREQUAL "RE2") build_re2() + elseif("${DEPENDENCY_NAME}" STREQUAL "Snappy") + build_snappy() elseif("${DEPENDENCY_NAME}" STREQUAL "Thrift") build_thrift() elseif("${DEPENDENCY_NAME}" STREQUAL "utf8proc") build_utf8proc() elseif("${DEPENDENCY_NAME}" STREQUAL "ZLIB") build_zlib() - elseif("${DEPENDENCY_NAME}" STREQUAL "ZSTD") + elseif("${DEPENDENCY_NAME}" STREQUAL "zstd") build_zstd() else() message(FATAL_ERROR "Unknown thirdparty dependency to build: ${DEPENDENCY_NAME}") endif() endmacro() -macro(resolve_dependency DEPENDENCY_NAME) - if(${DEPENDENCY_NAME}_SOURCE STREQUAL "AUTO") - find_package(${DEPENDENCY_NAME} MODULE) - if(NOT ${${DEPENDENCY_NAME}_FOUND}) - build_dependency(${DEPENDENCY_NAME}) - endif() - elseif(${DEPENDENCY_NAME}_SOURCE STREQUAL "BUNDLED") - build_dependency(${DEPENDENCY_NAME}) - elseif(${DEPENDENCY_NAME}_SOURCE STREQUAL "SYSTEM") - find_package(${DEPENDENCY_NAME} REQUIRED) +# Find modules are needed by the consumer in case of a static build, or if the +# linkage is PUBLIC or INTERFACE. +macro(provide_find_module DEPENDENCY_NAME) + set(module_ "${CMAKE_SOURCE_DIR}/cmake_modules/Find${DEPENDENCY_NAME}.cmake") + if(EXISTS "${module_}") + message(STATUS "Providing cmake module for ${DEPENDENCY_NAME}") + install(FILES "${module_}" DESTINATION "${ARROW_CMAKE_INSTALL_DIR}") endif() + unset(module_) endmacro() -macro(resolve_dependency_with_version DEPENDENCY_NAME REQUIRED_VERSION) +macro(resolve_dependency DEPENDENCY_NAME) + set(options) + set(one_value_args REQUIRED_VERSION) + cmake_parse_arguments(ARG + "${options}" + "${one_value_args}" + "${multi_value_args}" + ${ARGN}) + if(ARG_UNPARSED_ARGUMENTS) + message(SEND_ERROR "Error: unrecognized arguments: ${ARG_UNPARSED_ARGUMENTS}") + endif() + if(${DEPENDENCY_NAME}_SOURCE STREQUAL "AUTO") - find_package(${DEPENDENCY_NAME} ${REQUIRED_VERSION} MODULE) - if(NOT ${${DEPENDENCY_NAME}_FOUND}) + if(ARG_REQUIRED_VERSION) + find_package(${DEPENDENCY_NAME} ${ARG_REQUIRED_VERSION} MODULE) + else() + find_package(${DEPENDENCY_NAME} MODULE) + endif() + if(${${DEPENDENCY_NAME}_FOUND}) + set(${DEPENDENCY_NAME}_SOURCE "SYSTEM") + else() build_dependency(${DEPENDENCY_NAME}) + set(${DEPENDENCY_NAME}_SOURCE "BUNDLED") endif() elseif(${DEPENDENCY_NAME}_SOURCE STREQUAL "BUNDLED") build_dependency(${DEPENDENCY_NAME}) elseif(${DEPENDENCY_NAME}_SOURCE STREQUAL "SYSTEM") - find_package(${DEPENDENCY_NAME} ${REQUIRED_VERSION} REQUIRED) + if(ARG_REQUIRED_VERSION) + find_package(${DEPENDENCY_NAME} ${ARG_REQUIRED_VERSION} REQUIRED) + else() + find_package(${DEPENDENCY_NAME} REQUIRED) + endif() + endif() + if(${DEPENDENCY_NAME}_SOURCE STREQUAL "SYSTEM") + provide_find_module(${DEPENDENCY_NAME}) + list(APPEND ARROW_SYSTEM_DEPENDENCIES ${DEPENDENCY_NAME}) endif() endmacro() @@ -847,31 +877,7 @@ macro(build_snappy) endmacro() if(ARROW_WITH_SNAPPY) - if(Snappy_SOURCE STREQUAL "AUTO") - # Normally *Config.cmake files reside in /usr/lib/cmake but Snappy - # errornously places them in ${CMAKE_ROOT}/Modules/ - # This is fixed in 1.1.7 but fedora (30) still installs into the wrong - # location. - # https://bugzilla.redhat.com/show_bug.cgi?id=1679727 - # https://src.fedoraproject.org/rpms/snappy/pull-request/1 - find_package(Snappy QUIET HINTS "${CMAKE_ROOT}/Modules/") - if(NOT Snappy_FOUND) - find_package(SnappyAlt) - endif() - if(NOT Snappy_FOUND AND NOT SnappyAlt_FOUND) - build_snappy() - endif() - elseif(Snappy_SOURCE STREQUAL "BUNDLED") - build_snappy() - elseif(Snappy_SOURCE STREQUAL "SYSTEM") - # SnappyConfig.cmake is not installed on Ubuntu/Debian - # TODO: Make a bug report upstream - find_package(Snappy HINTS "${CMAKE_ROOT}/Modules/") - if(NOT Snappy_FOUND) - find_package(SnappyAlt REQUIRED) - endif() - endif() - + resolve_dependency(Snappy) # TODO: Don't use global includes but rather target_include_directories get_target_property(SNAPPY_INCLUDE_DIRS Snappy::snappy INTERFACE_INCLUDE_DIRECTORIES) include_directories(SYSTEM ${SNAPPY_INCLUDE_DIRS}) @@ -990,6 +996,7 @@ if(ARROW_USE_OPENSSL) INTERFACE_INCLUDE_DIRECTORIES "${OPENSSL_INCLUDE_DIR}") endif() + list(APPEND ARROW_SYSTEM_DEPENDENCIES "OpenSSL") include_directories(SYSTEM ${OPENSSL_INCLUDE_DIR}) else() @@ -1239,7 +1246,7 @@ if(ARROW_WITH_THRIFT) # to build Boost, so don't look again if already found. if(NOT Thrift_FOUND AND NOT THRIFT_FOUND) # Thrift c++ code generated by 0.13 requires 0.11 or greater - resolve_dependency_with_version(Thrift 0.11.0) + resolve_dependency(Thrift REQUIRED_VERSION 0.11.0) endif() # TODO: Don't use global includes but rather target_include_directories include_directories(SYSTEM ${THRIFT_INCLUDE_DIR}) @@ -1339,7 +1346,7 @@ if(ARROW_WITH_PROTOBUF) else() set(ARROW_PROTOBUF_REQUIRED_VERSION "2.6.1") endif() - resolve_dependency_with_version(Protobuf ${ARROW_PROTOBUF_REQUIRED_VERSION}) + resolve_dependency(Protobuf REQUIRED_VERSION ${ARROW_PROTOBUF_REQUIRED_VERSION}) if(ARROW_PROTOBUF_USE_SHARED AND MSVC) add_definitions(-DPROTOBUF_USE_DLLS) @@ -1982,7 +1989,7 @@ macro(build_zstd) endmacro() if(ARROW_WITH_ZSTD) - resolve_dependency(ZSTD) + resolve_dependency(zstd) if(TARGET zstd::libzstd) set(ARROW_ZSTD_LIBZSTD zstd::libzstd) @@ -2186,6 +2193,7 @@ macro(build_cares) set_target_properties(c-ares::cares PROPERTIES IMPORTED_LOCATION "${CARES_STATIC_LIB}" INTERFACE_INCLUDE_DIRECTORIES "${CARES_INCLUDE_DIR}") + add_dependencies(c-ares::cares cares_ep) set(CARES_VENDORED TRUE) diff --git a/cpp/examples/minimal_build/README.md b/cpp/examples/minimal_build/README.md index b79d20c2a0e2..9f889f6adda1 100644 --- a/cpp/examples/minimal_build/README.md +++ b/cpp/examples/minimal_build/README.md @@ -30,20 +30,14 @@ a file named `test.arrow`. ## Running the example -You can run this simple example using `Docker` and the given `Dockerfile`, -which installs a minimal Ubuntu image with a basic C++ toolchain. +You can run this simple example using [Docker Compose][docker-compose] +and the given `docker-compose.yml` and dockerfiles, which installs a +minimal Ubuntu image with a basic C++ toolchain. -Just open a terminal in this directory and, assuming `$ARROW_ROOT` points to -the Arrow C++ source tree (either a stable release or a git checkout), run -the following commands: +Just open a terminal in this directory and run the following commands: ```bash - -# Build the Docker image -docker build -t arrow_cpp_minimal . - -# Run the example inside the image -docker run -it -v $PWD:/io -v $ARROW_ROOT:/arrow arrow_cpp_minimal /io/run.sh +docker-compose run --rm minimal ``` Note that this example mounts two volumes inside the Docker image: @@ -53,12 +47,12 @@ Note that this example mounts two volumes inside the Docker image: ## Statically-linked builds We've provided an example build configuration here with CMake to show how to -create a statically-linked executable . +create a statically-linked executable with bundled dependencies. To run it on Linux, you can use the above Docker image: ```bash -docker run -it -v $PWD:/io -v $ARROW_ROOT:/arrow arrow_cpp_minimal /io/run_static.sh +docker-compose run --rm static ``` On macOS, you can use the `run_static.sh` but you must set some environment @@ -67,8 +61,8 @@ variables to point the script to your Arrow checkout, for example: ```bash export ARROW_DIR=path/to/arrow-clone export EXAMPLE_DIR=$ARROW_DIR/cpp/examples/minimal_build -export ARROW_BUILD_DIR=`pwd`/arrow-build -export EXAMPLE_BUILD_DIR=`pwd`/example +export ARROW_BUILD_DIR=$(pwd)/arrow-build +export EXAMPLE_BUILD_DIR=$(pwd)/example ./run_static.sh ``` @@ -78,4 +72,17 @@ Studio's command line tools enabled and CMake and ninja build in the path: ``` call run_static.bat -``` \ No newline at end of file +``` + +### Static linking against system libraries + +You can also use static libraries of Arrow's dependencies from the +system. To run this configuration, set +`ARROW_DEPENDENCY_SOURCE=SYSTEM` for `run_static.sh`. You can use +`docker-compose` for this too: + +```bash +docker-compose run --rm static-system-dependency +``` + +[docker-compose]: https://docs.docker.com/compose/ diff --git a/cpp/examples/minimal_build/docker-compose.yml b/cpp/examples/minimal_build/docker-compose.yml new file mode 100644 index 000000000000..6e2dcef81213 --- /dev/null +++ b/cpp/examples/minimal_build/docker-compose.yml @@ -0,0 +1,51 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +version: '3.5' + +services: + minimal: + build: + context: . + dockerfile: minimal.dockerfile + volumes: + - ../../../:/arrow:delegated + - .:/io:delegated + command: + - "/io/run.sh" + + static: + build: + context: . + dockerfile: minimal.dockerfile + volumes: + - ../../../:/arrow:delegated + - .:/io:delegated + command: + - "/io/run_static.sh" + + static-system-dependency: + build: + context: . + dockerfile: system_dependency.dockerfile + environment: + ARROW_DEPENDENCY_SOURCE: "SYSTEM" + volumes: + - ../../../:/arrow:delegated + - .:/io:delegated + command: + - "/io/run_static.sh" diff --git a/cpp/examples/minimal_build/Dockerfile b/cpp/examples/minimal_build/minimal.dockerfile similarity index 93% rename from cpp/examples/minimal_build/Dockerfile rename to cpp/examples/minimal_build/minimal.dockerfile index 6f00f475bdc3..95f73e9a5490 100644 --- a/cpp/examples/minimal_build/Dockerfile +++ b/cpp/examples/minimal_build/minimal.dockerfile @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. -FROM ubuntu:bionic +FROM ubuntu:focal ENV DEBIAN_FRONTEND=noninteractive @@ -23,4 +23,4 @@ RUN apt-get update -y -q && \ apt-get install -y -q --no-install-recommends \ build-essential \ cmake && \ - apt-get clean && rm -rf /var/lib/apt/lists* + apt-get clean && rm -rf /var/lib/apt/lists* diff --git a/cpp/examples/minimal_build/run_static.sh b/cpp/examples/minimal_build/run_static.sh index 941a29f31866..05804a0366cc 100755 --- a/cpp/examples/minimal_build/run_static.sh +++ b/cpp/examples/minimal_build/run_static.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information @@ -23,6 +23,8 @@ set -e : ${ARROW_BUILD_DIR:=/build/arrow} : ${EXAMPLE_BUILD_DIR:=/build/example} +: ${ARROW_DEPENDENCY_SOURCE:=BUNDLED} + echo echo "==" echo "== Building Arrow C++ library" @@ -35,12 +37,13 @@ pushd $ARROW_BUILD_DIR NPROC=$(nproc) cmake $ARROW_DIR/cpp \ - -DARROW_DEPENDENCY_SOURCE=BUNDLED \ -DARROW_BUILD_SHARED=OFF \ -DARROW_BUILD_STATIC=ON \ -DARROW_COMPUTE=ON \ -DARROW_CSV=ON \ -DARROW_DATASET=ON \ + -DARROW_DEPENDENCY_SOURCE=${ARROW_DEPENDENCY_SOURCE} \ + -DARROW_DEPENDENCY_USE_SHARED=OFF \ -DARROW_FILESYSTEM=ON \ -DARROW_HDFS=ON \ -DARROW_JEMALLOC=ON \ @@ -54,6 +57,7 @@ cmake $ARROW_DIR/cpp \ -DARROW_WITH_SNAPPY=ON \ -DARROW_WITH_ZLIB=ON \ -DARROW_WITH_ZSTD=ON \ + -DORC_SOURCE=BUNDLED \ $ARROW_CMAKE_OPTIONS make -j$NPROC @@ -81,6 +85,6 @@ echo "== Running example project" echo "==" echo -cd $EXAMPLE_DIR +pushd $EXAMPLE_DIR ${EXAMPLE_BUILD_DIR}/arrow_example diff --git a/cpp/examples/minimal_build/system_dependency.dockerfile b/cpp/examples/minimal_build/system_dependency.dockerfile new file mode 100644 index 000000000000..6615530bee15 --- /dev/null +++ b/cpp/examples/minimal_build/system_dependency.dockerfile @@ -0,0 +1,42 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +FROM ubuntu:focal + +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt-get update -y -q && \ + apt-get install -y -q --no-install-recommends \ + build-essential \ + cmake \ + libboost-filesystem-dev \ + libboost-regex-dev \ + libboost-system-dev \ + libbrotli-dev \ + libbz2-dev \ + libgflags-dev \ + liblz4-dev \ + libprotobuf-dev \ + libprotoc-dev \ + libsnappy-dev \ + libthrift-dev \ + libutf8proc-dev \ + libzstd-dev \ + protobuf-compiler \ + rapidjson-dev \ + zlib1g-dev && \ + apt-get clean && rm -rf /var/lib/apt/lists* diff --git a/cpp/src/arrow/ArrowConfig.cmake.in b/cpp/src/arrow/ArrowConfig.cmake.in index c4b35d5aba36..0cf8a85f570a 100644 --- a/cpp/src/arrow/ArrowConfig.cmake.in +++ b/cpp/src/arrow/ArrowConfig.cmake.in @@ -34,14 +34,40 @@ set(ARROW_VERSION "@ARROW_VERSION@") set(ARROW_SO_VERSION "@ARROW_SO_VERSION@") set(ARROW_FULL_SO_VERSION "@ARROW_FULL_SO_VERSION@") +set(ARROW_LIBRARY_PATH_SUFFIXES "@ARROW_LIBRARY_PATH_SUFFIXES@") +set(ARROW_INCLUDE_PATH_SUFFIXES "@ARROW_INCLUDE_PATH_SUFFIXES@") +set(ARROW_SYSTEM_DEPENDENCIES "@ARROW_SYSTEM_DEPENDENCIES@") + include("${CMAKE_CURRENT_LIST_DIR}/ArrowOptions.cmake") +include(CMakeFindDependencyMacro) + # Load targets only once. If we load targets multiple times, CMake reports # already existent target error. if(NOT (TARGET arrow_shared OR TARGET arrow_static)) include("${CMAKE_CURRENT_LIST_DIR}/ArrowTargets.cmake") if(TARGET arrow_static) + set(CMAKE_THREAD_PREFER_PTHREAD TRUE) + set(THREADS_PREFER_PTHREAD_FLAG TRUE) + find_dependency(Threads) + + if(DEFINED CMAKE_MODULE_PATH) + set(_CMAKE_MODULE_PATH_OLD ${CMAKE_MODULE_PATH}) + endif() + set(CMAKE_MODULE_PATH "${CMAKE_CURRENT_LIST_DIR}") + + foreach(_DEPENDENCY ${ARROW_SYSTEM_DEPENDENCIES}) + find_dependency(${_DEPENDENCY}) + endforeach() + + if(DEFINED _CMAKE_MODULE_PATH_OLD) + set(CMAKE_MODULE_PATH ${_CMAKE_MODULE_PATH_OLD}) + unset(_CMAKE_MODULE_PATH_OLD) + else() + unset(CMAKE_MODULE_PATH) + endif() + get_property(arrow_static_loc TARGET arrow_static PROPERTY LOCATION) get_filename_component(arrow_lib_dir ${arrow_static_loc} DIRECTORY) diff --git a/cpp/src/arrow/gpu/CMakeLists.txt b/cpp/src/arrow/gpu/CMakeLists.txt index c73ae19a41fe..a31b4be6ed1d 100644 --- a/cpp/src/arrow/gpu/CMakeLists.txt +++ b/cpp/src/arrow/gpu/CMakeLists.txt @@ -69,7 +69,13 @@ install(FILES "${CMAKE_CURRENT_BINARY_DIR}/cuda_version.h" arrow_install_all_headers("arrow/gpu") -set(ARROW_CUDA_TEST_LINK_LIBS arrow_cuda_shared ${ARROW_TEST_LINK_LIBS}) +if(ARROW_BUILD_SHARED) + set(ARROW_CUDA_LIBRARY arrow_cuda_shared) +else() + set(ARROW_CUDA_LIBRARY arrow_cuda_static) +endif() + +set(ARROW_CUDA_TEST_LINK_LIBS ${ARROW_CUDA_LIBRARY} ${ARROW_TEST_LINK_LIBS}) if(ARROW_BUILD_TESTS) add_arrow_test(cuda_test STATIC_LINK_LIBS ${ARROW_CUDA_TEST_LINK_LIBS} NO_VALGRIND) @@ -77,7 +83,7 @@ endif() if(ARROW_BUILD_BENCHMARKS) cuda_add_executable(arrow-cuda-benchmark cuda_benchmark.cc) - target_link_libraries(arrow-cuda-benchmark arrow_cuda_shared GTest::gtest + target_link_libraries(arrow-cuda-benchmark ${ARROW_CUDA_LIBRARY} GTest::gtest ${ARROW_BENCHMARK_LINK_LIBS}) add_dependencies(arrow_cuda-benchmarks arrow-cuda-benchmark) endif() diff --git a/cpp/src/gandiva/jni/CMakeLists.txt b/cpp/src/gandiva/jni/CMakeLists.txt index 24ceb6355917..b456d5f3def3 100644 --- a/cpp/src/gandiva/jni/CMakeLists.txt +++ b/cpp/src/gandiva/jni/CMakeLists.txt @@ -97,11 +97,13 @@ add_arrow_lib(gandiva_jni add_dependencies(gandiva ${GANDIVA_JNI_LIBRARIES}) -# filter out everything that is not needed for the jni bridge -# statically linked stdc++ has conflicts with stdc++ loaded by other libraries. -if(NOT APPLE) - set_target_properties( - gandiva_jni_shared - PROPERTIES LINK_FLAGS - "-Wl,--version-script=${CMAKE_SOURCE_DIR}/src/gandiva/jni/symbols.map") +if(ARROW_BUILD_SHARED) + # filter out everything that is not needed for the jni bridge + # statically linked stdc++ has conflicts with stdc++ loaded by other libraries. + if(NOT APPLE) + set_target_properties( + gandiva_jni_shared + PROPERTIES LINK_FLAGS + "-Wl,--version-script=${CMAKE_SOURCE_DIR}/src/gandiva/jni/symbols.map") + endif() endif() diff --git a/cpp/src/jni/orc/CMakeLists.txt b/cpp/src/jni/orc/CMakeLists.txt index 68da11f81f57..eceda5294947 100644 --- a/cpp/src/jni/orc/CMakeLists.txt +++ b/cpp/src/jni/orc/CMakeLists.txt @@ -31,10 +31,11 @@ set(JNI_HEADERS_DIR "${CMAKE_CURRENT_BINARY_DIR}/generated") add_subdirectory(../../../../java/adapter/orc ./java) -set(ARROW_BUILD_STATIC OFF) - add_arrow_lib(arrow_orc_jni BUILD_SHARED + ON + BUILD_STATIC + OFF SOURCES jni_wrapper.cpp OUTPUTS diff --git a/dev/tasks/cpp-examples/github.linux.yml b/dev/tasks/cpp-examples/github.linux.yml new file mode 100644 index 000000000000..717d3c44302a --- /dev/null +++ b/dev/tasks/cpp-examples/github.linux.yml @@ -0,0 +1,46 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# NOTE: must set "Crossbow" as name to have the badge links working in the +# github comment reports! +name: Crossbow + +on: + push: + branches: + - "*-github-*" + +jobs: + test: + name: C++ Example + runs-on: ubuntu-latest + steps: + - name: Checkout Arrow + shell: bash + run: | + git clone --no-checkout {{ arrow.remote }} arrow + git -C arrow fetch -t {{ arrow.remote }} {{ arrow.branch }} + git -C arrow checkout FETCH_HEAD + git -C arrow submodule update --init --recursive + - name: Free Up Disk Space + shell: bash + run: arrow/ci/scripts/util_cleanup.sh + - name: Run + shell: bash + run: | + cd arrow/cpp/examples/{{ type }} + docker-compose run --rm {{ run }} diff --git a/dev/tasks/linux-packages/apache-arrow/debian.ubuntu-xenial/libarrow-dev.install b/dev/tasks/linux-packages/apache-arrow/debian.ubuntu-xenial/libarrow-dev.install index 443e70cf0c1b..549cecef3880 100644 --- a/dev/tasks/linux-packages/apache-arrow/debian.ubuntu-xenial/libarrow-dev.install +++ b/dev/tasks/linux-packages/apache-arrow/debian.ubuntu-xenial/libarrow-dev.install @@ -3,6 +3,11 @@ usr/lib/*/cmake/arrow/ArrowConfig*.cmake usr/lib/*/cmake/arrow/ArrowOptions.cmake usr/lib/*/cmake/arrow/ArrowTargets*.cmake usr/lib/*/cmake/arrow/FindArrow.cmake +usr/lib/*/cmake/arrow/FindBrotli.cmake +usr/lib/*/cmake/arrow/FindLz4.cmake +usr/lib/*/cmake/arrow/FindSnappy.cmake +usr/lib/*/cmake/arrow/Findutf8proc.cmake +usr/lib/*/cmake/arrow/Findzstd.cmake usr/lib/*/cmake/arrow/arrow-config.cmake usr/lib/*/libarrow.a usr/lib/*/libarrow.so diff --git a/dev/tasks/linux-packages/apache-arrow/debian.ubuntu-xenial/libgandiva-dev.install b/dev/tasks/linux-packages/apache-arrow/debian.ubuntu-xenial/libgandiva-dev.install index 26e7e76fb380..7dabb79d0ddd 100644 --- a/dev/tasks/linux-packages/apache-arrow/debian.ubuntu-xenial/libgandiva-dev.install +++ b/dev/tasks/linux-packages/apache-arrow/debian.ubuntu-xenial/libgandiva-dev.install @@ -2,6 +2,7 @@ usr/include/gandiva/ usr/lib/*/cmake/arrow/GandivaConfig*.cmake usr/lib/*/cmake/arrow/GandivaTargets*.cmake usr/lib/*/cmake/arrow/FindGandiva.cmake +usr/lib/*/cmake/arrow/FindRE2.cmake usr/lib/*/libgandiva.a usr/lib/*/libgandiva.so usr/lib/*/pkgconfig/gandiva.pc diff --git a/dev/tasks/linux-packages/apache-arrow/debian/libarrow-dev.install b/dev/tasks/linux-packages/apache-arrow/debian/libarrow-dev.install index 443e70cf0c1b..549cecef3880 100644 --- a/dev/tasks/linux-packages/apache-arrow/debian/libarrow-dev.install +++ b/dev/tasks/linux-packages/apache-arrow/debian/libarrow-dev.install @@ -3,6 +3,11 @@ usr/lib/*/cmake/arrow/ArrowConfig*.cmake usr/lib/*/cmake/arrow/ArrowOptions.cmake usr/lib/*/cmake/arrow/ArrowTargets*.cmake usr/lib/*/cmake/arrow/FindArrow.cmake +usr/lib/*/cmake/arrow/FindBrotli.cmake +usr/lib/*/cmake/arrow/FindLz4.cmake +usr/lib/*/cmake/arrow/FindSnappy.cmake +usr/lib/*/cmake/arrow/Findutf8proc.cmake +usr/lib/*/cmake/arrow/Findzstd.cmake usr/lib/*/cmake/arrow/arrow-config.cmake usr/lib/*/libarrow.a usr/lib/*/libarrow.so diff --git a/dev/tasks/linux-packages/apache-arrow/debian/libgandiva-dev.install b/dev/tasks/linux-packages/apache-arrow/debian/libgandiva-dev.install index 26e7e76fb380..7dabb79d0ddd 100644 --- a/dev/tasks/linux-packages/apache-arrow/debian/libgandiva-dev.install +++ b/dev/tasks/linux-packages/apache-arrow/debian/libgandiva-dev.install @@ -2,6 +2,7 @@ usr/include/gandiva/ usr/lib/*/cmake/arrow/GandivaConfig*.cmake usr/lib/*/cmake/arrow/GandivaTargets*.cmake usr/lib/*/cmake/arrow/FindGandiva.cmake +usr/lib/*/cmake/arrow/FindRE2.cmake usr/lib/*/libgandiva.a usr/lib/*/libgandiva.so usr/lib/*/pkgconfig/gandiva.pc diff --git a/dev/tasks/linux-packages/apache-arrow/yum/arrow.spec.in b/dev/tasks/linux-packages/apache-arrow/yum/arrow.spec.in index 74248b3a33d0..6af80dab06b1 100644 --- a/dev/tasks/linux-packages/apache-arrow/yum/arrow.spec.in +++ b/dev/tasks/linux-packages/apache-arrow/yum/arrow.spec.in @@ -44,7 +44,9 @@ %define use_glib (%{_centos_ver} >= 7) %define use_meson (%{_centos_ver} >= 8) +%define have_brotli (%{_centos_ver} >= 7) %define have_rapidjson (%{_centos_ver} == 7) +%define have_re2 (%{_centos_ver} >= 8) %define have_utf8proc (%{_centos_ver} == 7) Name: @PACKAGE@ @@ -59,6 +61,8 @@ Source0: https://dist.apache.org/repos/dist/release/@PACKAGE@/@PACKAGE@-%{versio BuildRequires: bison %if %{_centos_ver} >= 7 BuildRequires: boost%{boost_version}-devel +%endif +%if %{have_brotli} BuildRequires: brotli-devel %endif BuildRequires: bzip2-devel @@ -79,7 +83,7 @@ BuildRequires: python%{python_version}-numpy %if %{have_rapidjson} BuildRequires: rapidjson-devel %endif -%if %{_centos_ver} >= 8 +%if %{have_re2} BuildRequires: re2-devel %endif BuildRequires: snappy-devel @@ -218,6 +222,8 @@ License: Apache-2.0 Requires: boost%{boost_version}-system Requires: boost%{boost_version}-filesystem Requires: boost%{boost_version}-regex +%endif +%if %{have_brotli} Requires: brotli %endif %if %{_centos_ver} >= 7 @@ -226,7 +232,7 @@ Requires: glog %endif Requires: libzstd Requires: lz4 -%if %{_centos_ver} >= 8 +%if %{have_re2} Requires: re2 %endif Requires: snappy @@ -265,6 +271,15 @@ Libraries and header files for Apache Arrow C++. %{_libdir}/cmake/arrow/ArrowOptions.cmake %{_libdir}/cmake/arrow/ArrowTargets*.cmake %{_libdir}/cmake/arrow/FindArrow.cmake +%if %{have_brotli} +%{_libdir}/cmake/arrow/FindBrotli.cmake +%endif +%{_libdir}/cmake/arrow/FindLz4.cmake +%{_libdir}/cmake/arrow/FindSnappy.cmake +%if %{have_utf8proc} +%{_libdir}/cmake/arrow/Findutf8proc.cmake +%endif +%{_libdir}/cmake/arrow/Findzstd.cmake %{_libdir}/cmake/arrow/arrow-config.cmake %{_libdir}/libarrow.a %{_libdir}/libarrow.so @@ -374,6 +389,9 @@ Libraries and header files for Gandiva. %{_libdir}/cmake/arrow/GandivaConfig*.cmake %{_libdir}/cmake/arrow/GandivaTargets*.cmake %{_libdir}/cmake/arrow/FindGandiva.cmake +%if %{have_re2} +%{_libdir}/cmake/arrow/FindRE2.cmake +%endif %{_libdir}/libgandiva.a %{_libdir}/libgandiva.so %{_libdir}/pkgconfig/gandiva.pc diff --git a/dev/tasks/python-wheels/win-build.bat b/dev/tasks/python-wheels/win-build.bat index 957d68d3d5f0..449834a83603 100644 --- a/dev/tasks/python-wheels/win-build.bat +++ b/dev/tasks/python-wheels/win-build.bat @@ -65,12 +65,14 @@ cmake -G "%GENERATOR%" ^ -DARROW_DATASET=ON ^ -DARROW_FLIGHT=ON ^ -DARROW_GANDIVA=OFF ^ + -DARROW_LZ4_USE_SHARED=OFF ^ -DARROW_MIMALLOC=ON ^ -DARROW_PARQUET=ON ^ -DARROW_PYTHON=ON ^ + -DARROW_SNAPPY_USE_SHARED=OFF ^ -DARROW_VERBOSE_THIRDPARTY_BUILD=ON ^ -DBrotli_SOURCE=BUNDLED ^ - -DZSTD_SOURCE=BUNDLED ^ + -Dzstd_SOURCE=BUNDLED ^ -Dutf8proc_SOURCE=BUNDLED ^ .. || exit /B cmake --build . --target install --config Release || exit /B diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml index 3fc9ab17f393..a064a420a4e6 100644 --- a/dev/tasks/tasks.yml +++ b/dev/tasks/tasks.yml @@ -78,6 +78,12 @@ groups: - test-*spark* - test-*turbodbc* + example: + - example-* + + example-cpp: + - example-*cpp* + verify-rc: - verify-rc-* @@ -109,6 +115,7 @@ groups: - homebrew-r-autobrew - nuget - test-* + - example-* - wheel-* tasks: @@ -1844,3 +1851,17 @@ tasks: template: docker-tests/github.linux.yml params: run: conda-cpp-hiveserver2 + + example-cpp-minimal-build-static: + ci: github + template: cpp-examples/github.linux.yml + params: + type: minimal_build + run: static + + example-cpp-minimal-build-static-system-dependency: + ci: github + template: cpp-examples/github.linux.yml + params: + type: minimal_build + run: static-system-dependency diff --git a/python/manylinux1/build_arrow.sh b/python/manylinux1/build_arrow.sh index 3abbfc718a81..5ffd25c5009f 100755 --- a/python/manylinux1/build_arrow.sh +++ b/python/manylinux1/build_arrow.sh @@ -103,11 +103,11 @@ cmake \ -DARROW_BUILD_TESTS=OFF \ -DARROW_DATASET=${BUILD_ARROW_DATASET} \ -DARROW_DEPENDENCY_SOURCE="SYSTEM" \ + -DARROW_DEPENDENCY_USE_SHARED=OFF \ -DARROW_FLIGHT=${BUILD_ARROW_FLIGHT} \ -DARROW_GANDIVA_JAVA=OFF \ -DARROW_GANDIVA_PC_CXX_FLAGS="-isystem;/opt/rh/devtoolset-2/root/usr/include/c++/4.8.2;-isystem;/opt/rh/devtoolset-2/root/usr/include/c++/4.8.2/x86_64-CentOS-linux/" \ -DARROW_GANDIVA=${BUILD_ARROW_GANDIVA} \ - -DARROW_GRPC_USE_SHARED=OFF \ -DARROW_HDFS=ON \ -DARROW_JEMALLOC=ON \ -DARROW_ORC=OFF \ @@ -125,7 +125,6 @@ cmake \ -DARROW_WITH_SNAPPY=ON \ -DARROW_WITH_ZLIB=ON \ -DARROW_WITH_ZSTD=ON \ - -DARROW_ZSTD_USE_SHARED=OFF \ -DBoost_NAMESPACE=arrow_boost \ -DBOOST_ROOT=/arrow_boost_dist \ -DCMAKE_INSTALL_LIBDIR=lib \ diff --git a/python/manylinux201x/build_arrow.sh b/python/manylinux201x/build_arrow.sh index 50141c3fc9dc..2f5134ba2f1c 100755 --- a/python/manylinux201x/build_arrow.sh +++ b/python/manylinux201x/build_arrow.sh @@ -104,11 +104,11 @@ PATH="${CPYTHON_PATH}/bin:${PATH}" cmake \ -DARROW_BUILD_TESTS=OFF \ -DARROW_DATASET=${BUILD_ARROW_DATASET} \ -DARROW_DEPENDENCY_SOURCE="SYSTEM" \ + -DARROW_DEPENDENCY_USE_SHARED=OFF \ -DARROW_FLIGHT=${BUILD_ARROW_FLIGHT} \ -DARROW_GANDIVA_JAVA=OFF \ -DARROW_GANDIVA_PC_CXX_FLAGS="-isystem;/opt/rh/devtoolset-8/root/usr/include/c++/8/;-isystem;/opt/rh/devtoolset-8/root/usr/include/c++/8/x86_64-redhat-linux/" \ -DARROW_GANDIVA=${BUILD_ARROW_GANDIVA} \ - -DARROW_GRPC_USE_SHARED=OFF \ -DARROW_HDFS=ON \ -DARROW_JEMALLOC=ON \ -DARROW_ORC=OFF \ @@ -126,7 +126,6 @@ PATH="${CPYTHON_PATH}/bin:${PATH}" cmake \ -DARROW_WITH_SNAPPY=ON \ -DARROW_WITH_ZLIB=ON \ -DARROW_WITH_ZSTD=ON \ - -DARROW_ZSTD_USE_SHARED=OFF \ -DBoost_NAMESPACE=arrow_boost \ -DBOOST_ROOT=/arrow_boost_dist \ -DCMAKE_BUILD_TYPE=Release \ From 52cb9cb19da4f0b52801f84757d37c8281165e85 Mon Sep 17 00:00:00 2001 From: Sagnik Chakraborty Date: Thu, 6 Aug 2020 18:46:38 +0530 Subject: [PATCH 004/460] ARROW-9607: [C++][Gandiva] Add bitwise_and(), bitwise_or() and bitwise_not() functions for integers Closes #7872 from sagnikc-dremio/bit and squashes the following commits: 2a29a9a8e ARROW-9607: Add bitwise_and(), bitwise_or() and bitwise_not() functions for integers Authored-by: Sagnik Chakraborty Signed-off-by: Praveen --- .../gandiva/function_registry_arithmetic.cc | 8 ++++++ cpp/src/gandiva/precompiled/arithmetic_ops.cc | 13 +++++++++ .../precompiled/arithmetic_ops_test.cc | 27 +++++++++++++++++++ cpp/src/gandiva/precompiled/types.h | 7 +++++ 4 files changed, 55 insertions(+) diff --git a/cpp/src/gandiva/function_registry_arithmetic.cc b/cpp/src/gandiva/function_registry_arithmetic.cc index dca273e8c365..0f8ffd34bd82 100644 --- a/cpp/src/gandiva/function_registry_arithmetic.cc +++ b/cpp/src/gandiva/function_registry_arithmetic.cc @@ -77,6 +77,14 @@ std::vector GetArithmeticFunctionRegistry() { BINARY_SYMMETRIC_UNSAFE_NULL_IF_NULL(div, {}, float32), BINARY_SYMMETRIC_UNSAFE_NULL_IF_NULL(div, {}, float64), + // bitwise operators + BINARY_SYMMETRIC_SAFE_NULL_IF_NULL(bitwise_and, {}, int32), + BINARY_SYMMETRIC_SAFE_NULL_IF_NULL(bitwise_and, {}, int64), + BINARY_SYMMETRIC_SAFE_NULL_IF_NULL(bitwise_or, {}, int32), + BINARY_SYMMETRIC_SAFE_NULL_IF_NULL(bitwise_or, {}, int64), + UNARY_SAFE_NULL_IF_NULL(bitwise_not, {}, int32, int32), + UNARY_SAFE_NULL_IF_NULL(bitwise_not, {}, int64, int64), + // compare functions BINARY_RELATIONAL_BOOL_FN(equal, ({"eq", "same"})), BINARY_RELATIONAL_BOOL_FN(not_equal, {}), diff --git a/cpp/src/gandiva/precompiled/arithmetic_ops.cc b/cpp/src/gandiva/precompiled/arithmetic_ops.cc index afeb2f9466bf..21bab714ab98 100644 --- a/cpp/src/gandiva/precompiled/arithmetic_ops.cc +++ b/cpp/src/gandiva/precompiled/arithmetic_ops.cc @@ -67,6 +67,10 @@ extern "C" { NUMERIC_TYPES(BINARY_SYMMETRIC, add, +) NUMERIC_TYPES(BINARY_SYMMETRIC, subtract, -) NUMERIC_TYPES(BINARY_SYMMETRIC, multiply, *) +BINARY_SYMMETRIC(bitwise_and, int32, &) +BINARY_SYMMETRIC(bitwise_and, int64, &) +BINARY_SYMMETRIC(bitwise_or, int32, |) +BINARY_SYMMETRIC(bitwise_or, int64, |) #undef BINARY_SYMMETRIC @@ -234,6 +238,15 @@ DIV_FLOAT(float64) #undef DIV_FLOAT +#define BITWISE_NOT(TYPE) \ + FORCE_INLINE \ + gdv_##TYPE bitwise_not_##TYPE(gdv_##TYPE in) { return static_cast(~in); } + +BITWISE_NOT(int32) +BITWISE_NOT(int64) + +#undef BITWISE_NOT + #undef DATE_FUNCTION #undef DATE_TYPES #undef NUMERIC_BOOL_DATE_TYPES diff --git a/cpp/src/gandiva/precompiled/arithmetic_ops_test.cc b/cpp/src/gandiva/precompiled/arithmetic_ops_test.cc index a62020e69b86..f8b9937a2c5f 100644 --- a/cpp/src/gandiva/precompiled/arithmetic_ops_test.cc +++ b/cpp/src/gandiva/precompiled/arithmetic_ops_test.cc @@ -101,4 +101,31 @@ TEST(TestArithmeticOps, TestDiv) { context.Reset(); } +TEST(TestArithmeticOps, TestBitwiseOps) { + // bitwise AND + EXPECT_EQ(bitwise_and_int32_int32(0x0147D, 0x17159), 0x01059); + EXPECT_EQ(bitwise_and_int32_int32(0xFFFFFFCC, 0x00000297), 0x00000284); + EXPECT_EQ(bitwise_and_int32_int32(0x000, 0x285), 0x000); + EXPECT_EQ(bitwise_and_int64_int64(0x563672F83, 0x0D9FCF85B), 0x041642803); + EXPECT_EQ(bitwise_and_int64_int64(0xFFFFFFFFFFDA8F6A, 0xFFFFFFFFFFFF791C), + 0xFFFFFFFFFFDA0908); + EXPECT_EQ(bitwise_and_int64_int64(0x6A5B1, 0x00000), 0x00000); + + // bitwise OR + EXPECT_EQ(bitwise_or_int32_int32(0x0147D, 0x17159), 0x1757D); + EXPECT_EQ(bitwise_or_int32_int32(0xFFFFFFCC, 0x00000297), 0xFFFFFFDF); + EXPECT_EQ(bitwise_or_int32_int32(0x000, 0x285), 0x285); + EXPECT_EQ(bitwise_or_int64_int64(0x563672F83, 0x0D9FCF85B), 0x5FBFFFFDB); + EXPECT_EQ(bitwise_or_int64_int64(0xFFFFFFFFFFDA8F6A, 0xFFFFFFFFFFFF791C), + 0xFFFFFFFFFFFFFF7E); + EXPECT_EQ(bitwise_or_int64_int64(0x6A5B1, 0x00000), 0x6A5B1); + + // bitwise NOT + EXPECT_EQ(bitwise_not_int32(0x00017159), 0xFFFE8EA6); + EXPECT_EQ(bitwise_not_int32(0xFFFFF226), 0x00000DD9); + EXPECT_EQ(bitwise_not_int64(0x000000008BCAE9B4), 0xFFFFFFFF7435164B); + EXPECT_EQ(bitwise_not_int64(0xFFFFFF966C8D7997), 0x0000006993728668); + EXPECT_EQ(bitwise_not_int64(0x0000000000000000), 0xFFFFFFFFFFFFFFFF); +} + } // namespace gandiva diff --git a/cpp/src/gandiva/precompiled/types.h b/cpp/src/gandiva/precompiled/types.h index 77f1589a73b2..25a8b1c4106c 100644 --- a/cpp/src/gandiva/precompiled/types.h +++ b/cpp/src/gandiva/precompiled/types.h @@ -152,6 +152,13 @@ gdv_float64 log10_int64(gdv_int64); gdv_float64 log10_float32(gdv_float32); gdv_float64 log10_float64(gdv_float64); +gdv_int32 bitwise_and_int32_int32(gdv_int32 in1, gdv_int32 in2); +gdv_int64 bitwise_and_int64_int64(gdv_int64 in1, gdv_int64 in2); +gdv_int32 bitwise_or_int32_int32(gdv_int32 in1, gdv_int32 in2); +gdv_int64 bitwise_or_int64_int64(gdv_int64 in1, gdv_int64 in2); +gdv_int32 bitwise_not_int32(gdv_int32); +gdv_int64 bitwise_not_int64(gdv_int64); + gdv_float64 power_float64_float64(gdv_float64, gdv_float64); gdv_float64 log_int32_int32(gdv_int64 context, gdv_int32 base, gdv_int32 value); From 3332247b006ea76db8c3ad18d382fc73ce79e1f5 Mon Sep 17 00:00:00 2001 From: alamb Date: Thu, 6 Aug 2020 08:40:47 -0600 Subject: [PATCH 005/460] ARROW-9656: [Rust] [DataFusion] Better error messages for unsupported EXTERNAL TABLE types Previously, the error message for an invalid CREATE EXTERNAL TABLE command told the user there was a problem but not how to fix it. This PR improves the error message and adds some parser tests. Before this PR: ``` CREATE EXTERNAL TABLE repro STORED AS ARROW LOCATION 'repro.arrow'; ParserError(ParserError("Expected fileformat, found: ARROW")) ``` After this PR: ``` CREATE EXTERNAL TABLE repro STORED AS ARROW LOCATION 'repro.arrow'; ParserError(ParserError("Expected one of PARQUET, NDJSON, or CSV, found: ARROW")) ``` Closes #7905 from alamb/alamb/better-parser-errors Authored-by: alamb Signed-off-by: Andy Grove --- rust/datafusion/src/sql/parser.rs | 97 +++++++++++++++++++++++++++++-- 1 file changed, 92 insertions(+), 5 deletions(-) diff --git a/rust/datafusion/src/sql/parser.rs b/rust/datafusion/src/sql/parser.rs index 6a74f7fdff66..613fdf73c0f2 100644 --- a/rust/datafusion/src/sql/parser.rs +++ b/rust/datafusion/src/sql/parser.rs @@ -34,7 +34,7 @@ macro_rules! parser_err { } /// Types of files to parse as DataFrames -#[derive(Debug, Clone)] +#[derive(Debug, Clone, PartialEq)] pub enum FileType { /// Newline-delimited JSON NdJson, @@ -44,10 +44,10 @@ pub enum FileType { CSV, } -/// DataFrame Statement's representations. +/// DataFusion Statement representations. /// /// Tokens parsed by `DFParser` are converted into these values. -#[derive(Debug, Clone)] +#[derive(Debug, Clone, PartialEq)] pub enum Statement { /// ANSI SQL AST node Statement(SQLStatement), @@ -238,9 +238,9 @@ impl DFParser { "PARQUET" => Ok(FileType::Parquet), "NDJSON" => Ok(FileType::NdJson), "CSV" => Ok(FileType::CSV), - _ => self.expected("fileformat", Token::Word(w)), + _ => self.expected("one of PARQUET, NDJSON, or CSV", Token::Word(w)), }, - unexpected => self.expected("fileformat", unexpected), + unexpected => self.expected("one of PARQUET, NDJSON, or CSV", unexpected), } } @@ -259,3 +259,90 @@ impl DFParser { & self.consume_token("ROW") } } + +#[cfg(test)] +mod tests { + use super::*; + use sqlparser::ast::{DataType, Ident}; + + fn expect_parse_ok(sql: &str, expected: Statement) -> Result<(), ParserError> { + let statements = DFParser::parse_sql(sql)?; + assert_eq!( + statements.len(), + 1, + "Expected to parse exactly one statement" + ); + assert_eq!(statements[0], expected); + Ok(()) + } + + /// Parses sql and asserts that the expected error message was found + fn expect_parse_error(sql: &str, expected_error: &str) -> Result<(), ParserError> { + match DFParser::parse_sql(sql) { + Ok(statements) => { + assert!( + false, + "Expected parse error for '{}', but was successful: {:?}", + sql, statements + ); + } + Err(e) => { + let error_message = e.to_string(); + assert!( + error_message.contains(expected_error), + "Expected error '{}' not found in actual error '{}'", + expected_error, + error_message + ); + } + } + Ok(()) + } + + fn make_column_def(name: impl Into, data_type: DataType) -> ColumnDef { + ColumnDef { + name: Ident { + value: name.into(), + quote_style: None, + }, + data_type, + collation: None, + options: vec![], + } + } + + #[test] + fn create_external_table() -> Result<(), ParserError> { + // positive case + let sql = "CREATE EXTERNAL TABLE t(c1 int) STORED AS CSV LOCATION 'foo.csv'"; + let expected = Statement::CreateExternalTable { + name: "t".into(), + columns: vec![make_column_def("c1", DataType::Int)], + file_type: FileType::CSV, + has_header: false, + location: "foo.csv".into(), + }; + expect_parse_ok(sql, expected)?; + + // positive case: it is ok for parquet files not to have columns specified + let sql = "CREATE EXTERNAL TABLE t STORED AS PARQUET LOCATION 'foo.parquet'"; + let expected = Statement::CreateExternalTable { + name: "t".into(), + columns: vec![], + file_type: FileType::Parquet, + has_header: false, + location: "foo.parquet".into(), + }; + expect_parse_ok(sql, expected)?; + + // Error cases: Invalid type + let sql = + "CREATE EXTERNAL TABLE t(c1 int) STORED AS UNKNOWN_TYPE LOCATION 'foo.csv'"; + expect_parse_error( + sql, + "Expected one of PARQUET, NDJSON, or CSV, found: UNKNOWN_TYPE", + )?; + + Ok(()) + } +} From 0c914f8870ad9d027b3944b8820ce4a6b64d208c Mon Sep 17 00:00:00 2001 From: wqc200 Date: Thu, 6 Aug 2020 08:41:57 -0600 Subject: [PATCH 006/460] ARROW-9632: [Rust] add a func "new" for ExecutionContextSchemaProvider I use ExecutionContextSchemaProvider in outside app, so i add keyword "pub" for ExecutionContextSchemaProvider, and add a new func "new" for ExecutionContextSchemaProvider. I add keyword "pub" for build_schema also. Closes #7899 from wqc200/master2 Authored-by: wqc200 Signed-off-by: Andy Grove --- rust/datafusion/src/execution/context.rs | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/rust/datafusion/src/execution/context.rs b/rust/datafusion/src/execution/context.rs index b9f444a15004..6f0f65ed1aef 100644 --- a/rust/datafusion/src/execution/context.rs +++ b/rust/datafusion/src/execution/context.rs @@ -195,7 +195,8 @@ impl ExecutionContext { &self.scalar_functions } - fn build_schema(&self, columns: &Vec) -> Result { + /// Get schema from columns + pub fn build_schema(&self, columns: &Vec) -> Result { let mut fields = Vec::new(); for column in columns { @@ -667,11 +668,25 @@ impl ExecutionContext { } } -struct ExecutionContextSchemaProvider<'a> { +/// Get schema and scalar functions for execution context +pub struct ExecutionContextSchemaProvider<'a> { datasources: &'a HashMap>, scalar_functions: &'a HashMap>, } +impl<'a> ExecutionContextSchemaProvider<'a> { + /// Create a new ExecutionContextSchemaProvider based on data sources and scalar functions + pub fn new( + datasources: &'a HashMap>, + scalar_functions: &'a HashMap>, + ) -> Self { + ExecutionContextSchemaProvider { + datasources, + scalar_functions, + } + } +} + impl SchemaProvider for ExecutionContextSchemaProvider<'_> { fn get_table_meta(&self, name: &str) -> Option { self.datasources.get(name).map(|ds| ds.schema().clone()) From 62dfa114d6683172927fab40fa6c4ddabae8fef4 Mon Sep 17 00:00:00 2001 From: Yordan Pavlov Date: Thu, 6 Aug 2020 08:44:16 -0600 Subject: [PATCH 007/460] ARROW-9523 [Rust] Improve filter kernel performance MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The filter kernel located here https://github.com/apache/arrow/blob/master/rust/arrow/src/compute/kernels/filter.rs currently has the following performance: filter old u8 low selectivity time: [1.7782 ms 1.7790 ms 1.7801 ms] filter old u8 high selectivity time: [815.58 us 816.58 us 817.57 us] filter old u8 w NULLs low selectivity time: [1.8131 ms 1.8231 ms 1.8336 ms] filter old u8 w NULLs high selectivity time: [817.41 us 820.01 us 823.05 us] I have been working on a new implementation which performs between approximately 17 and 550 times faster depending mostly on filter selectivity. Here are the benchmark results: filter u8 low selectivity time: [107.26 us 108.24 us 109.58 us] filter u8 high selectivity time: [4.7854 us 4.8050 us 4.8276 us] filter context u8 low selectivity time: [102.59 us 102.93 us 103.38 us] filter context u8 high selectivity time: [1.4709 us 1.4760 us 1.4823 us] filter context u8 w NULLs low selectivity time: [130.48 us 131.00 us 131.65 us] filter context u8 w NULLs high selectivity time: [2.0520 us 2.0818 us 2.1137 us] filter context f32 low selectivity time: [117.26 us 118.58 us 120.13 us] filter context f32 high selectivity time: [1.7895 us 1.7919 us 1.7942 us] This new implementation is based on a few key ideas: (1) if the data array being filtered doesn't have a null bitmap, no time should be wasted to copy or create a null bitmap in the resulting filtered data array - this is implemented using the CopyNullBit trait which has a no-op implementation and an actual implementation (2) when the filter is highly selective, e.g. only a small number of values from the data array are selected, the filter implementation should efficiently skip entire batches of 0s in the filter array - this is implemented by transmuting the filter array to u64 which allows to quickly check and skip entire batches of 64 bits  (3) when an entire record batch is filtered, any computation which only depends on the filter array is done once and then shared for filtering all the data arrays in the record batch - this is implemented using the FilterContext struct This pull request also implements support for filtering dictionary arrays.  @paddyhoran @andygrove Closes #7798 from yordan-pavlov/improve_filter_kernel_perf Lead-authored-by: Yordan Pavlov Co-authored-by: Yordan Pavlov <64363766+yordan-pavlov@users.noreply.github.com> Signed-off-by: Andy Grove --- rust/arrow/Cargo.toml | 4 + rust/arrow/benches/filter_kernels.rs | 152 ++++++ rust/arrow/src/array/data.rs | 1 + rust/arrow/src/buffer.rs | 2 + rust/arrow/src/compute/kernels/filter.rs | 656 +++++++++++++++++++---- rust/arrow/src/util/bit_util.rs | 56 +- 6 files changed, 758 insertions(+), 113 deletions(-) create mode 100644 rust/arrow/benches/filter_kernels.rs diff --git a/rust/arrow/Cargo.toml b/rust/arrow/Cargo.toml index 0501c210fa53..b8dad6f093d7 100644 --- a/rust/arrow/Cargo.toml +++ b/rust/arrow/Cargo.toml @@ -85,6 +85,10 @@ harness = false name = "comparison_kernels" harness = false +[[bench]] +name = "filter_kernels" +harness = false + [[bench]] name = "take_kernels" harness = false diff --git a/rust/arrow/benches/filter_kernels.rs b/rust/arrow/benches/filter_kernels.rs new file mode 100644 index 000000000000..75c04352c0a7 --- /dev/null +++ b/rust/arrow/benches/filter_kernels.rs @@ -0,0 +1,152 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use arrow::array::*; +use arrow::compute::{filter, FilterContext}; +use arrow::datatypes::ArrowNumericType; +use criterion::{criterion_group, criterion_main, Criterion}; + +fn create_primitive_array(size: usize, value_fn: F) -> PrimitiveArray +where + T: ArrowNumericType, + F: Fn(usize) -> T::Native, +{ + let mut builder = PrimitiveArray::::builder(size); + for i in 0..size { + builder.append_value(value_fn(i)).unwrap(); + } + builder.finish() +} + +fn create_u8_array_with_nulls(size: usize) -> UInt8Array { + let mut builder = UInt8Builder::new(size); + for i in 0..size { + if i % 2 == 0 { + builder.append_value(1).unwrap(); + } else { + builder.append_null().unwrap(); + } + } + builder.finish() +} + +fn create_bool_array(size: usize, value_fn: F) -> BooleanArray +where + F: Fn(usize) -> bool, +{ + let mut builder = BooleanBuilder::new(size); + for i in 0..size { + builder.append_value(value_fn(i)).unwrap(); + } + builder.finish() +} + +fn bench_filter_u8(data_array: &UInt8Array, filter_array: &BooleanArray) { + filter( + criterion::black_box(data_array), + criterion::black_box(filter_array), + ) + .unwrap(); +} + +// fn bench_filter_f32(data_array: &Float32Array, filter_array: &BooleanArray) { +// filter(criterion::black_box(data_array), criterion::black_box(filter_array)).unwrap(); +// } + +fn bench_filter_context_u8(data_array: &UInt8Array, filter_context: &FilterContext) { + filter_context + .filter(criterion::black_box(data_array)) + .unwrap(); +} + +fn bench_filter_context_f32(data_array: &Float32Array, filter_context: &FilterContext) { + filter_context + .filter(criterion::black_box(data_array)) + .unwrap(); +} + +fn add_benchmark(c: &mut Criterion) { + let size = 65536; + let filter_array = create_bool_array(size, |i| match i % 2 { + 0 => true, + _ => false, + }); + let sparse_filter_array = create_bool_array(size, |i| match i % 8000 { + 0 => true, + _ => false, + }); + let dense_filter_array = create_bool_array(size, |i| match i % 8000 { + 0 => false, + _ => true, + }); + + let filter_context = FilterContext::new(&filter_array).unwrap(); + let sparse_filter_context = FilterContext::new(&sparse_filter_array).unwrap(); + let dense_filter_context = FilterContext::new(&dense_filter_array).unwrap(); + + let data_array = create_primitive_array(size, |i| match i % 2 { + 0 => 1, + _ => 0, + }); + c.bench_function("filter u8 low selectivity", |b| { + b.iter(|| bench_filter_u8(&data_array, &filter_array)) + }); + c.bench_function("filter u8 high selectivity", |b| { + b.iter(|| bench_filter_u8(&data_array, &sparse_filter_array)) + }); + c.bench_function("filter u8 very low selectivity", |b| { + b.iter(|| bench_filter_u8(&data_array, &dense_filter_array)) + }); + + c.bench_function("filter context u8 low selectivity", |b| { + b.iter(|| bench_filter_context_u8(&data_array, &filter_context)) + }); + c.bench_function("filter context u8 high selectivity", |b| { + b.iter(|| bench_filter_context_u8(&data_array, &sparse_filter_context)) + }); + c.bench_function("filter context u8 very low selectivity", |b| { + b.iter(|| bench_filter_context_u8(&data_array, &dense_filter_context)) + }); + + let data_array = create_u8_array_with_nulls(size); + c.bench_function("filter context u8 w NULLs low selectivity", |b| { + b.iter(|| bench_filter_context_u8(&data_array, &filter_context)) + }); + c.bench_function("filter context u8 w NULLs high selectivity", |b| { + b.iter(|| bench_filter_context_u8(&data_array, &sparse_filter_context)) + }); + c.bench_function("filter context u8 w NULLs very low selectivity", |b| { + b.iter(|| bench_filter_context_u8(&data_array, &dense_filter_context)) + }); + + let data_array = create_primitive_array(size, |i| match i % 2 { + 0 => 1.0, + _ => 0.0, + }); + c.bench_function("filter context f32 low selectivity", |b| { + b.iter(|| bench_filter_context_f32(&data_array, &filter_context)) + }); + c.bench_function("filter context f32 high selectivity", |b| { + b.iter(|| bench_filter_context_f32(&data_array, &sparse_filter_context)) + }); + c.bench_function("filter context f32 very low selectivity", |b| { + b.iter(|| bench_filter_context_f32(&data_array, &dense_filter_context)) + }); +} + +criterion_group!(benches, add_benchmark); +criterion_main!(benches); diff --git a/rust/arrow/src/array/data.rs b/rust/arrow/src/array/data.rs index 5f6e5dc9ae72..f1e32c57d988 100644 --- a/rust/arrow/src/array/data.rs +++ b/rust/arrow/src/array/data.rs @@ -152,6 +152,7 @@ impl ArrayData { } /// Returns the offset of this array + #[inline] pub fn offset(&self) -> usize { self.offset } diff --git a/rust/arrow/src/buffer.rs b/rust/arrow/src/buffer.rs index ca01f1d150bf..33a0af9e4bd6 100644 --- a/rust/arrow/src/buffer.rs +++ b/rust/arrow/src/buffer.rs @@ -545,10 +545,12 @@ impl MutableBuffer { /// /// Note that this should be used cautiously, and the returned pointer should not be /// stored anywhere, to avoid dangling pointers. + #[inline] pub fn raw_data(&self) -> *const u8 { self.data } + #[inline] pub fn raw_data_mut(&mut self) -> *mut u8 { self.data } diff --git a/rust/arrow/src/compute/kernels/filter.rs b/rust/arrow/src/compute/kernels/filter.rs index 52e12cfef19c..98d70f05cedd 100644 --- a/rust/arrow/src/compute/kernels/filter.rs +++ b/rust/arrow/src/compute/kernels/filter.rs @@ -17,139 +17,465 @@ //! Defines miscellaneous array kernels. -use std::sync::Arc; - use crate::array::*; use crate::datatypes::{ArrowNumericType, DataType, TimeUnit}; use crate::error::{ArrowError, Result}; +use crate::record_batch::RecordBatch; +use crate::{ + bitmap::Bitmap, + buffer::{Buffer, MutableBuffer}, + util::bit_util, +}; +use std::{mem, sync::Arc}; -/// Helper function to perform boolean lambda function on values from two arrays. -fn bool_op( - left: &PrimitiveArray, - right: &PrimitiveArray, - op: F, -) -> Result -where - T: ArrowNumericType, - F: Fn(Option, Option) -> bool, -{ - if left.len() != right.len() { - return Err(ArrowError::ComputeError( - "Cannot perform math operation on arrays of different length".to_string(), - )); +/// trait for copying filtered null bitmap bits +trait CopyNullBit { + fn copy_null_bit(&mut self, source_index: usize); + fn copy_null_bits(&mut self, source_index: usize, count: usize); + fn null_count(&self) -> usize; + fn null_buffer(&mut self) -> Buffer; +} + +/// no-op null bitmap copy implementation, +/// used when the filtered data array doesn't have a null bitmap +struct NullBitNoop {} + +impl NullBitNoop { + fn new() -> Self { + NullBitNoop {} } - let mut b = BooleanArray::builder(left.len()); - for i in 0..left.len() { - let index = i; - let l = if left.is_null(i) { - None - } else { - Some(left.value(index)) - }; - let r = if right.is_null(i) { - None - } else { - Some(right.value(index)) - }; - b.append_value(op(l, r))?; +} + +impl CopyNullBit for NullBitNoop { + #[inline] + fn copy_null_bit(&mut self, _source_index: usize) { + // do nothing + } + + #[inline] + fn copy_null_bits(&mut self, _source_index: usize, _count: usize) { + // do nothing + } + + fn null_count(&self) -> usize { + 0 + } + + fn null_buffer(&mut self) -> Buffer { + Buffer::from([0u8; 0]) } - Ok(b.finish()) } -macro_rules! filter_array { - ($array:expr, $filter:expr, $array_type:ident) => {{ - let b = $array.as_any().downcast_ref::<$array_type>().unwrap(); - let mut builder = $array_type::builder(b.len()); - for i in 0..b.len() { - if $filter.value(i) { - if b.is_null(i) { - builder.append_null()?; - } else { - builder.append_value(b.value(i))?; - } - } - } - Ok(Arc::new(builder.finish())) - }}; +/// null bitmap copy implementation, +/// used when the filtered data array has a null bitmap +struct NullBitSetter<'a> { + target_buffer: MutableBuffer, + source_bytes: &'a [u8], + target_index: usize, + null_count: usize, } -/// Returns the array, taking only the elements matching the filter -pub fn filter(array: &Array, filter: &BooleanArray) -> Result { - match array.data_type() { - DataType::UInt8 => filter_array!(array, filter, UInt8Array), - DataType::UInt16 => filter_array!(array, filter, UInt16Array), - DataType::UInt32 => filter_array!(array, filter, UInt32Array), - DataType::UInt64 => filter_array!(array, filter, UInt64Array), - DataType::Int8 => filter_array!(array, filter, Int8Array), - DataType::Int16 => filter_array!(array, filter, Int16Array), - DataType::Int32 => filter_array!(array, filter, Int32Array), - DataType::Int64 => filter_array!(array, filter, Int64Array), - DataType::Float32 => filter_array!(array, filter, Float32Array), - DataType::Float64 => filter_array!(array, filter, Float64Array), - DataType::Boolean => filter_array!(array, filter, BooleanArray), - DataType::Date32(_) => filter_array!(array, filter, Date32Array), - DataType::Date64(_) => filter_array!(array, filter, Date64Array), - DataType::Time32(TimeUnit::Second) => { - filter_array!(array, filter, Time32SecondArray) - } - DataType::Time32(TimeUnit::Millisecond) => { - filter_array!(array, filter, Time32MillisecondArray) +impl<'a> NullBitSetter<'a> { + fn new(null_bitmap: &'a Bitmap) -> Self { + let null_bytes = null_bitmap.buffer_ref().data(); + // create null bitmap buffer with same length and initialize null bitmap buffer to 1s + let null_buffer = + MutableBuffer::new(null_bytes.len()).with_bitset(null_bytes.len(), true); + NullBitSetter { + source_bytes: null_bytes, + target_buffer: null_buffer, + target_index: 0, + null_count: 0, } - DataType::Time64(TimeUnit::Microsecond) => { - filter_array!(array, filter, Time64MicrosecondArray) - } - DataType::Time64(TimeUnit::Nanosecond) => { - filter_array!(array, filter, Time64NanosecondArray) - } - DataType::Duration(TimeUnit::Second) => { - filter_array!(array, filter, DurationSecondArray) - } - DataType::Duration(TimeUnit::Millisecond) => { - filter_array!(array, filter, DurationMillisecondArray) - } - DataType::Duration(TimeUnit::Microsecond) => { - filter_array!(array, filter, DurationMicrosecondArray) - } - DataType::Duration(TimeUnit::Nanosecond) => { - filter_array!(array, filter, DurationNanosecondArray) + } +} + +impl<'a> CopyNullBit for NullBitSetter<'a> { + #[inline] + fn copy_null_bit(&mut self, source_index: usize) { + if !bit_util::get_bit(self.source_bytes, source_index) { + bit_util::unset_bit(self.target_buffer.data_mut(), self.target_index); + self.null_count += 1; } - DataType::Timestamp(TimeUnit::Second, _) => { - filter_array!(array, filter, TimestampSecondArray) + self.target_index += 1; + } + + #[inline] + fn copy_null_bits(&mut self, source_index: usize, count: usize) { + for i in 0..count { + self.copy_null_bit(source_index + i); } - DataType::Timestamp(TimeUnit::Millisecond, _) => { - filter_array!(array, filter, TimestampMillisecondArray) + } + + fn null_count(&self) -> usize { + self.null_count + } + + fn null_buffer(&mut self) -> Buffer { + self.target_buffer.resize(self.target_index).unwrap(); + // use mem::replace to detach self.target_buffer from self so that it can be returned + let target_buffer = mem::replace(&mut self.target_buffer, MutableBuffer::new(0)); + target_buffer.freeze() + } +} + +fn get_null_bit_setter<'a>(data_array: &'a impl Array) -> Box { + if let Some(null_bitmap) = data_array.data_ref().null_bitmap() { + // only return an actual null bit copy implementation if null_bitmap is set + Box::new(NullBitSetter::new(null_bitmap)) + } else { + // otherwise return a no-op copy null bit implementation + // for improved performance when the filtered array doesn't contain NULLs + Box::new(NullBitNoop::new()) + } +} + +// transmute filter array to u64 +// - optimize filtering with highly selective filters by skipping entire batches of 64 filter bits +// - if the data array being filtered doesn't have a null bitmap, no time is wasted to copy a null bitmap +fn filter_array_impl( + filter_context: &FilterContext, + data_array: &impl Array, + array_type: DataType, + value_size: usize, +) -> Result { + if filter_context.filter_len > data_array.len() { + return Err(ArrowError::ComputeError( + "Filter array cannot be larger than data array".to_string(), + )); + } + let filtered_count = filter_context.filtered_count; + let filter_mask = &filter_context.filter_mask; + let filter_u64 = &filter_context.filter_u64; + let data_bytes = data_array.data_ref().buffers()[0].data(); + let mut target_buffer = MutableBuffer::new(filtered_count * value_size); + target_buffer.resize(filtered_count * value_size)?; + let target_bytes = target_buffer.data_mut(); + let mut target_byte_index: usize = 0; + let mut null_bit_setter = get_null_bit_setter(data_array); + let null_bit_setter = null_bit_setter.as_mut(); + let all_ones_batch = !0u64; + let data_array_offset = data_array.offset(); + + for (i, filter_batch) in filter_u64.iter().enumerate() { + // foreach u64 batch + let filter_batch = *filter_batch; + if filter_batch == 0 { + // if batch == 0: skip + continue; + } else if filter_batch == all_ones_batch { + // if batch == all 1s: copy all 64 values in one go + let data_index = (i * 64) + data_array_offset; + null_bit_setter.copy_null_bits(data_index, 64); + let data_byte_index = data_index * value_size; + let data_len = value_size * 64; + target_bytes[target_byte_index..(target_byte_index + data_len)] + .copy_from_slice( + &data_bytes[data_byte_index..(data_byte_index + data_len)], + ); + target_byte_index += data_len; + continue; } - DataType::Timestamp(TimeUnit::Microsecond, _) => { - filter_array!(array, filter, TimestampMicrosecondArray) + for (j, filter_mask) in filter_mask.iter().enumerate() { + // foreach bit in batch: + if (filter_batch & *filter_mask) != 0 { + let data_index = (i * 64) + j + data_array_offset; + null_bit_setter.copy_null_bit(data_index); + // if filter bit == 1: copy data value bytes + let data_byte_index = data_index * value_size; + target_bytes[target_byte_index..(target_byte_index + value_size)] + .copy_from_slice( + &data_bytes[data_byte_index..(data_byte_index + value_size)], + ); + target_byte_index += value_size; + } } - DataType::Timestamp(TimeUnit::Nanosecond, _) => { - filter_array!(array, filter, TimestampNanosecondArray) + } + + let mut array_data_builder = ArrayDataBuilder::new(array_type) + .len(filtered_count) + .add_buffer(target_buffer.freeze()); + if null_bit_setter.null_count() > 0 { + array_data_builder = array_data_builder + .null_count(null_bit_setter.null_count()) + .null_bit_buffer(null_bit_setter.null_buffer()); + } + + Ok(array_data_builder) +} + +/// FilterContext can be used to improve performance when +/// filtering multiple data arrays with the same filter array. +#[derive(Debug)] +pub struct FilterContext { + filter_u64: Vec, + filter_len: usize, + filtered_count: usize, + filter_mask: Vec, +} + +macro_rules! filter_primitive_array { + ($context:expr, $array:expr, $array_type:ident) => {{ + let input_array = $array.as_any().downcast_ref::<$array_type>().unwrap(); + let output_array = $context.filter_primitive_array(input_array)?; + Ok(Arc::new(output_array)) + }}; +} + +macro_rules! filter_dictionary_array { + ($context:expr, $array:expr, $array_type:ident) => {{ + let input_array = $array.as_any().downcast_ref::<$array_type>().unwrap(); + let output_array = $context.filter_dictionary_array(input_array)?; + Ok(Arc::new(output_array)) + }}; +} + +impl FilterContext { + /// Returns a new instance of FilterContext + pub fn new(filter_array: &BooleanArray) -> Result { + if filter_array.offset() > 0 { + return Err(ArrowError::ComputeError( + "Filter array cannot have offset > 0".to_string(), + )); } - DataType::Binary => { - let b = array.as_any().downcast_ref::().unwrap(); - let mut values: Vec<&[u8]> = Vec::with_capacity(b.len()); - for i in 0..b.len() { - if filter.value(i) { - values.push(b.value(i)); + let filter_mask: Vec = (0..64).map(|x| 1u64 << x).collect(); + let filter_bytes = filter_array.data_ref().buffers()[0].data(); + let filtered_count = bit_util::count_set_bits(filter_bytes); + // transmute filter_bytes to &[u64] + let mut u64_buffer = MutableBuffer::new(filter_bytes.len()); + u64_buffer + .write_bytes(filter_bytes, u64_buffer.capacity() - filter_bytes.len())?; + let filter_u64 = u64_buffer.typed_data_mut::().to_owned(); + Ok(FilterContext { + filter_u64, + filter_len: filter_array.len(), + filtered_count, + filter_mask, + }) + } + + /// Returns a new array, containing only the elements matching the filter + pub fn filter(&self, array: &Array) -> Result { + match array.data_type() { + DataType::UInt8 => filter_primitive_array!(self, array, UInt8Array), + DataType::UInt16 => filter_primitive_array!(self, array, UInt16Array), + DataType::UInt32 => filter_primitive_array!(self, array, UInt32Array), + DataType::UInt64 => filter_primitive_array!(self, array, UInt64Array), + DataType::Int8 => filter_primitive_array!(self, array, Int8Array), + DataType::Int16 => filter_primitive_array!(self, array, Int16Array), + DataType::Int32 => filter_primitive_array!(self, array, Int32Array), + DataType::Int64 => filter_primitive_array!(self, array, Int64Array), + DataType::Float32 => filter_primitive_array!(self, array, Float32Array), + DataType::Float64 => filter_primitive_array!(self, array, Float64Array), + DataType::Boolean => { + let input_array = array.as_any().downcast_ref::().unwrap(); + let mut builder = BooleanArray::builder(self.filtered_count); + for i in 0..self.filter_u64.len() { + // foreach u64 batch + let filter_batch = self.filter_u64[i]; + if filter_batch == 0 { + // if batch == 0: skip + continue; + } + for j in 0..64 { + // foreach bit in batch: + if (filter_batch & self.filter_mask[j]) != 0 { + let data_index = (i * 64) + j; + if input_array.is_null(data_index) { + builder.append_null()?; + } else { + builder.append_value(input_array.value(data_index))?; + } + } + } } + Ok(Arc::new(builder.finish())) + }, + DataType::Date32(_) => filter_primitive_array!(self, array, Date32Array), + DataType::Date64(_) => filter_primitive_array!(self, array, Date64Array), + DataType::Time32(TimeUnit::Second) => { + filter_primitive_array!(self, array, Time32SecondArray) } - Ok(Arc::new(BinaryArray::from(values))) - } - DataType::Utf8 => { - let b = array.as_any().downcast_ref::().unwrap(); - let mut values: Vec<&str> = Vec::with_capacity(b.len()); - for i in 0..b.len() { - if filter.value(i) { - values.push(b.value(i)); + DataType::Time32(TimeUnit::Millisecond) => { + filter_primitive_array!(self, array, Time32MillisecondArray) + } + DataType::Time64(TimeUnit::Microsecond) => { + filter_primitive_array!(self, array, Time64MicrosecondArray) + } + DataType::Time64(TimeUnit::Nanosecond) => { + filter_primitive_array!(self, array, Time64NanosecondArray) + } + DataType::Duration(TimeUnit::Second) => { + filter_primitive_array!(self, array, DurationSecondArray) + } + DataType::Duration(TimeUnit::Millisecond) => { + filter_primitive_array!(self, array, DurationMillisecondArray) + } + DataType::Duration(TimeUnit::Microsecond) => { + filter_primitive_array!(self, array, DurationMicrosecondArray) + } + DataType::Duration(TimeUnit::Nanosecond) => { + filter_primitive_array!(self, array, DurationNanosecondArray) + } + DataType::Timestamp(TimeUnit::Second, _) => { + filter_primitive_array!(self, array, TimestampSecondArray) + } + DataType::Timestamp(TimeUnit::Millisecond, _) => { + filter_primitive_array!(self, array, TimestampMillisecondArray) + } + DataType::Timestamp(TimeUnit::Microsecond, _) => { + filter_primitive_array!(self, array, TimestampMicrosecondArray) + } + DataType::Timestamp(TimeUnit::Nanosecond, _) => { + filter_primitive_array!(self, array, TimestampNanosecondArray) + } + DataType::Binary => { + let input_array = array.as_any().downcast_ref::().unwrap(); + let mut values: Vec<&[u8]> = Vec::with_capacity(self.filtered_count); + for i in 0..self.filter_u64.len() { + // foreach u64 batch + let filter_batch = self.filter_u64[i]; + if filter_batch == 0 { + // if batch == 0: skip + continue; + } + for j in 0..64 { + // foreach bit in batch: + if (filter_batch & self.filter_mask[j]) != 0 { + let data_index = (i * 64) + j; + values.push(input_array.value(data_index)); + } + } + } + Ok(Arc::new(BinaryArray::from(values))) + } + DataType::Utf8 => { + let input_array = array.as_any().downcast_ref::().unwrap(); + let mut values: Vec<&str> = Vec::with_capacity(self.filtered_count); + for i in 0..self.filter_u64.len() { + // foreach u64 batch + let filter_batch = self.filter_u64[i]; + if filter_batch == 0 { + // if batch == 0: skip + continue; + } + for j in 0..64 { + // foreach bit in batch: + if (filter_batch & self.filter_mask[j]) != 0 { + let data_index = (i * 64) + j; + values.push(input_array.value(data_index)); + } + } + } + Ok(Arc::new(StringArray::from(values))) + } + DataType::Dictionary(ref key_type, ref value_type) => match (key_type.as_ref(), value_type.as_ref()) { + (key_type, DataType::Utf8) => match key_type { + DataType::UInt8 => filter_dictionary_array!(self, array, UInt8DictionaryArray), + DataType::UInt16 => filter_dictionary_array!(self, array, UInt16DictionaryArray), + DataType::UInt32 => filter_dictionary_array!(self, array, UInt32DictionaryArray), + DataType::UInt64 => filter_dictionary_array!(self, array, UInt64DictionaryArray), + DataType::Int8 => filter_dictionary_array!(self, array, Int8DictionaryArray), + DataType::Int16 => filter_dictionary_array!(self, array, Int16DictionaryArray), + DataType::Int32 => filter_dictionary_array!(self, array, Int32DictionaryArray), + DataType::Int64 => filter_dictionary_array!(self, array, Int64DictionaryArray), + other => Err(ArrowError::ComputeError(format!( + "filter not supported for string dictionary with key of type {:?}", + other + ))) } + (key_type, value_type) => Err(ArrowError::ComputeError(format!( + "filter not supported for Dictionary({:?}, {:?})", + key_type, value_type + ))) } - Ok(Arc::new(StringArray::from(values))) + other => Err(ArrowError::ComputeError(format!( + "filter not supported for {:?}", + other + ))), } - other => Err(ArrowError::ComputeError(format!( - "filter not supported for {:?}", - other - ))), } + + /// Returns a new PrimitiveArray containing only those values from the array passed as the data_array parameter, + /// selected by the BooleanArray passed as the filter_array parameter + pub fn filter_primitive_array( + &self, + data_array: &PrimitiveArray, + ) -> Result> + where + T: ArrowNumericType, + { + let array_type = T::get_data_type(); + let value_size = mem::size_of::(); + let array_data_builder = + filter_array_impl(self, data_array, array_type, value_size)?; + let data = array_data_builder.build(); + Ok(PrimitiveArray::::from(data)) + } + + /// Returns a new DictionaryArray containing only those keys from the array passed as the data_array parameter, + /// selected by the BooleanArray passed as the filter_array parameter. The values are cloned from the data_array. + pub fn filter_dictionary_array( + &self, + data_array: &DictionaryArray, + ) -> Result> + where + T: ArrowNumericType, + { + let array_type = data_array.data_type().clone(); + let value_size = mem::size_of::(); + let mut array_data_builder = + filter_array_impl(self, data_array, array_type, value_size)?; + // copy dictionary values from input array + array_data_builder = + array_data_builder.add_child_data(data_array.values().data()); + let data = array_data_builder.build(); + Ok(DictionaryArray::::from(data)) + } +} + +/// Returns a new array, containing only the elements matching the filter. +pub fn filter(array: &Array, filter: &BooleanArray) -> Result { + FilterContext::new(filter)?.filter(array) +} + +/// Returns a new PrimitiveArray containing only those values from the array passed as the data_array parameter, +/// selected by the BooleanArray passed as the filter_array parameter +pub fn filter_primitive_array( + data_array: &PrimitiveArray, + filter_array: &BooleanArray, +) -> Result> +where + T: ArrowNumericType, +{ + FilterContext::new(filter_array)?.filter_primitive_array(data_array) +} + +/// Returns a new DictionaryArray containing only those keys from the array passed as the data_array parameter, +/// selected by the BooleanArray passed as the filter_array parameter. The values are cloned from the data_array. +pub fn filter_dictionary_array( + data_array: &DictionaryArray, + filter_array: &BooleanArray, +) -> Result> +where + T: ArrowNumericType, +{ + FilterContext::new(filter_array)?.filter_dictionary_array(data_array) +} + +/// Returns a new RecordBatch with arrays containing only values matching the filter. +/// The same FilterContext is re-used when filtering arrays in the RecordBatch for better performance. +pub fn filter_record_batch( + record_batch: &RecordBatch, + filter_array: &BooleanArray, +) -> Result { + let filter_context = FilterContext::new(filter_array)?; + let filtered_arrays = record_batch + .columns() + .iter() + .map(|a| filter_context.filter(a.as_ref())) + .collect::>>()?; + RecordBatch::try_new(record_batch.schema(), filtered_arrays) } #[cfg(test)] @@ -253,6 +579,73 @@ mod tests { assert_eq!(8, d.value(1)); } + #[test] + fn test_filter_array_slice() { + let a_slice = Int32Array::from(vec![5, 6, 7, 8, 9]).slice(1, 4); + let a = a_slice.as_ref(); + let b = BooleanArray::from(vec![true, false, false, true]); + // filtering with sliced filter array is not currently supported + // let b_slice = BooleanArray::from(vec![true, false, false, true, false]).slice(1, 4); + // let b = b_slice.as_any().downcast_ref().unwrap(); + let c = filter(a, &b).unwrap(); + let d = c.as_ref().as_any().downcast_ref::().unwrap(); + assert_eq!(2, d.len()); + assert_eq!(6, d.value(0)); + assert_eq!(9, d.value(1)); + } + + #[test] + fn test_filter_array_low_density() { + // this test exercises the all 0's branch of the filter algorithm + let mut data_values = (1..=65).into_iter().collect::>(); + let mut filter_values = (1..=65) + .into_iter() + .map(|i| match i % 65 { + 0 => true, + _ => false, + }) + .collect::>(); + // set up two more values after the batch + data_values.extend_from_slice(&[66, 67]); + filter_values.extend_from_slice(&[false, true]); + let a = Int32Array::from(data_values); + let b = BooleanArray::from(filter_values); + let c = filter(&a, &b).unwrap(); + let d = c.as_ref().as_any().downcast_ref::().unwrap(); + assert_eq!(2, d.len()); + assert_eq!(65, d.value(0)); + assert_eq!(67, d.value(1)); + } + + #[test] + fn test_filter_array_high_density() { + // this test exercises the all 1's branch of the filter algorithm + let mut data_values = (1..=65).into_iter().map(|x| Some(x)).collect::>(); + let mut filter_values = (1..=65) + .into_iter() + .map(|i| match i % 65 { + 0 => false, + _ => true, + }) + .collect::>(); + // set second data value to null + data_values[1] = None; + // set up two more values after the batch + data_values.extend_from_slice(&[Some(66), None, Some(67), None]); + filter_values.extend_from_slice(&[false, true, true, true]); + let a = Int32Array::from(data_values); + let b = BooleanArray::from(filter_values); + let c = filter(&a, &b).unwrap(); + let d = c.as_ref().as_any().downcast_ref::().unwrap(); + assert_eq!(67, d.len()); + assert_eq!(3, d.null_count()); + assert_eq!(1, d.value(0)); + assert_eq!(true, d.is_null(1)); + assert_eq!(64, d.value(63)); + assert_eq!(true, d.is_null(64)); + assert_eq!(67, d.value(65)); + } + #[test] fn test_filter_string_array() { let a = StringArray::from(vec!["hello", " ", "world", "!"]); @@ -273,4 +666,45 @@ mod tests { assert_eq!(1, d.len()); assert_eq!(true, d.is_null(0)); } + + #[test] + fn test_filter_array_slice_with_null() { + let a_slice = + Int32Array::from(vec![Some(5), None, Some(7), Some(8), Some(9)]).slice(1, 4); + let a = a_slice.as_ref(); + let b = BooleanArray::from(vec![true, false, false, true]); + // filtering with sliced filter array is not currently supported + // let b_slice = BooleanArray::from(vec![true, false, false, true, false]).slice(1, 4); + // let b = b_slice.as_any().downcast_ref().unwrap(); + let c = filter(a, &b).unwrap(); + let d = c.as_ref().as_any().downcast_ref::().unwrap(); + assert_eq!(2, d.len()); + assert_eq!(true, d.is_null(0)); + assert_eq!(false, d.is_null(1)); + assert_eq!(9, d.value(1)); + } + + #[test] + fn test_filter_dictionary_array() { + let values = vec![Some("hello"), None, Some("world"), Some("!")]; + let a: Int8DictionaryArray = values.iter().map(|&x| x).collect(); + let b = BooleanArray::from(vec![false, true, true, false]); + let c = filter(&a, &b).unwrap(); + let d = c + .as_ref() + .as_any() + .downcast_ref::() + .unwrap(); + let value_array = d.values(); + let values = value_array.as_any().downcast_ref::().unwrap(); + // values are cloned in the filtered dictionary array + assert_eq!(3, values.len()); + // but keys are filtered + assert_eq!(2, d.len()); + assert_eq!(true, d.is_null(0)); + assert_eq!( + "world", + values.value(d.keys().nth(1).unwrap().unwrap() as usize) + ); + } } diff --git a/rust/arrow/src/util/bit_util.rs b/rust/arrow/src/util/bit_util.rs index a2ada2c03237..d8ffa6f19c5d 100644 --- a/rust/arrow/src/util/bit_util.rs +++ b/rust/arrow/src/util/bit_util.rs @@ -68,7 +68,7 @@ pub unsafe fn get_bit_raw(data: *const u8, i: usize) -> bool { /// Sets bit at position `i` for `data` #[inline] pub fn set_bit(data: &mut [u8], i: usize) { - data[i >> 3] |= BIT_MASK[i & 7] + data[i >> 3] |= BIT_MASK[i & 7]; } /// Sets bit at position `i` for `data` @@ -79,7 +79,24 @@ pub fn set_bit(data: &mut [u8], i: usize) { /// responsible to guarantee that `i` is within bounds. #[inline] pub unsafe fn set_bit_raw(data: *mut u8, i: usize) { - *data.add(i >> 3) |= BIT_MASK[i & 7] + *data.add(i >> 3) |= BIT_MASK[i & 7]; +} + +/// Sets bit at position `i` for `data` to 0 +#[inline] +pub fn unset_bit(data: &mut [u8], i: usize) { + data[i >> 3] ^= BIT_MASK[i & 7]; +} + +/// Sets bit at position `i` for `data` to 0 +/// +/// # Safety +/// +/// Note this doesn't do any bound checking, for performance reason. The caller is +/// responsible to guarantee that `i` is within bounds. +#[inline] +pub unsafe fn unset_bit_raw(data: *mut u8, i: usize) { + *data.add(i >> 3) ^= BIT_MASK[i & 7]; } /// Sets bits in the non-inclusive range `start..end` for `data` @@ -257,6 +274,17 @@ mod tests { assert_eq!([0b00100101], b); } + #[test] + fn test_unset_bit() { + let mut b = [0b11111111]; + unset_bit(&mut b, 0); + assert_eq!([0b11111110], b); + unset_bit(&mut b, 2); + assert_eq!([0b11111010], b); + unset_bit(&mut b, 5); + assert_eq!([0b11011010], b); + } + #[test] fn test_set_bit_raw() { const NUM_BYTE: usize = 10; @@ -281,6 +309,30 @@ mod tests { } } + #[test] + fn test_unset_bit_raw() { + const NUM_BYTE: usize = 10; + let mut buf = vec![255; NUM_BYTE]; + let mut expected = vec![]; + let mut rng = thread_rng(); + for i in 0..8 * NUM_BYTE { + let b = rng.gen_bool(0.5); + expected.push(b); + if !b { + unsafe { + unset_bit_raw(buf.as_mut_ptr(), i); + } + } + } + + let raw_ptr = buf.as_ptr(); + for (i, b) in expected.iter().enumerate() { + unsafe { + assert_eq!(*b, get_bit_raw(raw_ptr, i)); + } + } + } + #[test] fn test_set_bits_raw() { const NUM_BYTE: usize = 64; From 7525286b87b148974badea71c136d2ae115386cd Mon Sep 17 00:00:00 2001 From: Benjamin Kietzman Date: Thu, 6 Aug 2020 12:10:42 -0400 Subject: [PATCH 008/460] ARROW-9573: [Python][Dataset] Provide read_table(ignore_prefixes=) Closes #7900 from bkietz/9573-expose-ignore_prefixes Authored-by: Benjamin Kietzman Signed-off-by: Benjamin Kietzman --- python/pyarrow/dataset.py | 2 +- python/pyarrow/parquet.py | 23 +++++++++++++++++++---- python/pyarrow/tests/test_parquet.py | 24 ++++++++++++++++++++++++ 3 files changed, 44 insertions(+), 5 deletions(-) diff --git a/python/pyarrow/dataset.py b/python/pyarrow/dataset.py index f4620d07662f..fd03aee71444 100644 --- a/python/pyarrow/dataset.py +++ b/python/pyarrow/dataset.py @@ -581,7 +581,7 @@ def dataset(source, schema=None, format=None, filesystem=None, files may be present in the Dataset (resulting in an error at scan time). ignore_prefixes : list, optional - Files matching one of those prefixes will be ignored by the + Files matching any of these prefixes will be ignored by the discovery process. This is matched to the basename of a path. By default this is ['.', '_']. Note that discovery happens only if a directory is passed as source. diff --git a/python/pyarrow/parquet.py b/python/pyarrow/parquet.py index 59c79ac1c16b..b5be07f778b8 100644 --- a/python/pyarrow/parquet.py +++ b/python/pyarrow/parquet.py @@ -1376,7 +1376,7 @@ class _ParquetDatasetV2: def __init__(self, path_or_paths, filesystem=None, filters=None, partitioning="hive", read_dictionary=None, buffer_size=None, - memory_map=False, **kwargs): + memory_map=False, ignore_prefixes=None, **kwargs): import pyarrow.dataset as ds import pyarrow.fs @@ -1430,7 +1430,8 @@ def __init__(self, path_or_paths, filesystem=None, filters=None, self._dataset = ds.dataset(path_or_paths, filesystem=filesystem, format=parquet_format, - partitioning=partitioning) + partitioning=partitioning, + ignore_prefixes=ignore_prefixes) @property def schema(self): @@ -1521,6 +1522,12 @@ def pieces(self): for all columns and not only the partition keys, enables different partitioning schemes, etc. Set to False to use the legacy behaviour. +ignore_prefixes : list, optional + Files matching any of these prefixes will be ignored by the + discovery process if use_legacy_dataset=False. + This is matched to the basename of a path. + By default this is ['.', '_']. + Note that discovery happens only if a directory is passed as source. filesystem : FileSystem, default None If nothing passed, paths assumed to be found in the local on-disk filesystem. @@ -1544,7 +1551,8 @@ def pieces(self): def read_table(source, columns=None, use_threads=True, metadata=None, use_pandas_metadata=False, memory_map=False, read_dictionary=None, filesystem=None, filters=None, - buffer_size=0, partitioning="hive", use_legacy_dataset=False): + buffer_size=0, partitioning="hive", use_legacy_dataset=False, + ignore_prefixes=None): if not use_legacy_dataset: if metadata is not None: raise ValueError( @@ -1562,6 +1570,7 @@ def read_table(source, columns=None, use_threads=True, metadata=None, read_dictionary=read_dictionary, buffer_size=buffer_size, filters=filters, + ignore_prefixes=ignore_prefixes, ) except ImportError: # fall back on ParquetFile for simple cases when pyarrow.dataset @@ -1585,6 +1594,11 @@ def read_table(source, columns=None, use_threads=True, metadata=None, return dataset.read(columns=columns, use_threads=use_threads, use_pandas_metadata=use_pandas_metadata) + if ignore_prefixes is not None: + raise ValueError( + "The 'ignore_prefixes' keyword is only supported when " + "use_legacy_dataset=False") + if _is_path_like(source): pf = ParquetDataset(source, metadata=metadata, memory_map=memory_map, read_dictionary=read_dictionary, @@ -1616,7 +1630,7 @@ def read_table(source, columns=None, use_threads=True, metadata=None, def read_pandas(source, columns=None, use_threads=True, memory_map=False, metadata=None, filters=None, buffer_size=0, - use_legacy_dataset=True): + use_legacy_dataset=True, ignore_prefixes=None): return read_table( source, columns=columns, @@ -1627,6 +1641,7 @@ def read_pandas(source, columns=None, use_threads=True, memory_map=False, buffer_size=buffer_size, use_pandas_metadata=True, use_legacy_dataset=use_legacy_dataset, + ignore_prefixes=ignore_prefixes ) diff --git a/python/pyarrow/tests/test_parquet.py b/python/pyarrow/tests/test_parquet.py index a24b1b34d8af..07af08fe48e9 100644 --- a/python/pyarrow/tests/test_parquet.py +++ b/python/pyarrow/tests/test_parquet.py @@ -2678,6 +2678,30 @@ def test_ignore_no_private_directories_path_list( _assert_dataset_paths(dataset, paths, use_legacy_dataset) +@pytest.mark.pandas +@parametrize_legacy_dataset_fixed +def test_ignore_custom_prefixes(tempdir, use_legacy_dataset): + # ARROW-9573 - allow override of default ignore_prefixes + part = ["xxx"] * 3 + ["yyy"] * 3 + table = pa.table([ + pa.array(range(len(part))), + pa.array(part).dictionary_encode(), + ], names=['index', '_part']) + + pq.write_to_dataset(table, str(tempdir), partition_cols=['_part']) + + private_duplicate = tempdir / '_private_duplicate' + private_duplicate.mkdir() + pq.write_to_dataset(table, str(private_duplicate), + partition_cols=['_part']) + + read = pq.read_table( + tempdir, use_legacy_dataset=use_legacy_dataset, + ignore_prefixes=['_private']) + + assert read.equals(table) + + @parametrize_legacy_dataset_fixed def test_empty_directory(tempdir, use_legacy_dataset): # ARROW-5310 - reading empty directory From fd6f5451d2bc4277f88cf42c2d7d6b71c503a380 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Fri, 7 Aug 2020 09:34:07 +0900 Subject: [PATCH 009/460] ARROW-9666: [Python][wheel][Windows] Fix wheel build for Windows It's caused by ARROW-9412 https://github.com/apache/arrow/pull/7842 . Closes #7914 from kou/python-wheel-windows-fix-link Authored-by: Sutou Kouhei Signed-off-by: Sutou Kouhei --- cpp/cmake_modules/DefineOptions.cmake | 10 +++++++++- cpp/cmake_modules/FindSnappy.cmake | 9 ++++----- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/cpp/cmake_modules/DefineOptions.cmake b/cpp/cmake_modules/DefineOptions.cmake index 555099fb4d6e..cf860cb41898 100644 --- a/cpp/cmake_modules/DefineOptions.cmake +++ b/cpp/cmake_modules/DefineOptions.cmake @@ -364,8 +364,16 @@ if("${CMAKE_SOURCE_DIR}" STREQUAL "${CMAKE_CURRENT_SOURCE_DIR}") define_option_string(RE2_MSVC_STATIC_LIB_SUFFIX "re2 static lib suffix used on Windows with MSVC" "_static") + if(DEFINED ENV{CONDA_PREFIX}) + # Conda package changes the output name. + # https://github.com/conda-forge/snappy-feedstock/blob/master/recipe/windows-static-lib-name.patch + set(SNAPPY_MSVC_STATIC_LIB_SUFFIX_DEFAULT "_static") + else() + set(SNAPPY_MSVC_STATIC_LIB_SUFFIX_DEFAULT "") + endif() define_option_string(SNAPPY_MSVC_STATIC_LIB_SUFFIX - "Snappy static lib suffix used on Windows with MSVC" "_static") + "Snappy static lib suffix used on Windows with MSVC" + "${SNAPPY_MSVC_STATIC_LIB_SUFFIX_DEFAULT}") define_option_string(LZ4_MSVC_STATIC_LIB_SUFFIX "Lz4 static lib suffix used on Windows with MSVC" "_static") diff --git a/cpp/cmake_modules/FindSnappy.cmake b/cpp/cmake_modules/FindSnappy.cmake index 7be7fb264d10..24fdf1031c7a 100644 --- a/cpp/cmake_modules/FindSnappy.cmake +++ b/cpp/cmake_modules/FindSnappy.cmake @@ -24,12 +24,11 @@ if(ARROW_SNAPPY_USE_SHARED) list(APPEND SNAPPY_LIB_NAMES "${CMAKE_SHARED_LIBRARY_PREFIX}snappy${CMAKE_SHARED_LIBRARY_SUFFIX}") else() - if(MSVC AND DEFINED ENV{CONDA_PREFIX}) - # Conda package changes the output name. - # https://github.com/conda-forge/snappy-feedstock/blob/master/recipe/windows-static-lib-name.patch - set(SNAPPY_LIB_NAMES "${CMAKE_STATIC_LIBRARY_PREFIX}snappy_static${CMAKE_STATIC_LIBRARY_SUFFIX}") + set(SNAPPY_STATIC_LIB_NAME_BASE "snappy") + if(MSVC) + set(SNAPPY_STATIC_LIB_NAME_BASE "${SNAPPY_STATIC_LIB_NAME_BASE}${SNAPPY_MSVC_STATIC_LIB_SUFFIX}") endif() - set(SNAPPY_LIB_NAMES "${CMAKE_STATIC_LIBRARY_PREFIX}snappy${CMAKE_STATIC_LIBRARY_SUFFIX}") + set(SNAPPY_LIB_NAMES "${CMAKE_STATIC_LIBRARY_PREFIX}${SNAPPY_STATIC_LIB_NAME_BASE}${CMAKE_STATIC_LIBRARY_SUFFIX}") endif() if(Snappy_ROOT) From 634fcd1f625ec53c1526e598db8bf996b4022be4 Mon Sep 17 00:00:00 2001 From: fredgan Date: Fri, 7 Aug 2020 14:45:31 +0200 Subject: [PATCH 010/460] ARROW-9548: [Go] Test output files are not removed correctly @sbinet Please review it Closes #7823 from fredgan/file_remove_9548 Authored-by: fredgan Signed-off-by: Sebastien Binet --- go/arrow/arrio/arrio_test.go | 12 ++++++++---- go/arrow/internal/arrjson/arrjson_test.go | 9 +++++++-- go/arrow/ipc/cmd/arrow-cat/main_test.go | 18 ++++++++++++++---- .../ipc/cmd/arrow-file-to-stream/main_test.go | 13 +++++++++---- .../arrow-json-integration-test/main_test.go | 17 ++++++++++------- go/arrow/ipc/cmd/arrow-ls/main_test.go | 18 ++++++++++++++---- .../ipc/cmd/arrow-stream-to-file/main_test.go | 12 ++++++++---- go/arrow/ipc/file_test.go | 9 +++++++-- go/arrow/ipc/stream_test.go | 9 +++++++-- 9 files changed, 84 insertions(+), 33 deletions(-) diff --git a/go/arrow/arrio/arrio_test.go b/go/arrow/arrio/arrio_test.go index 74f7f735c5da..783309b22803 100644 --- a/go/arrow/arrio/arrio_test.go +++ b/go/arrow/arrio/arrio_test.go @@ -67,6 +67,12 @@ func (k copyKind) check(t *testing.T, f *os.File, mem memory.Allocator, schema * func TestCopy(t *testing.T) { type kind int + tempDir, err := ioutil.TempDir("", "go-arrow-copy-") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(tempDir) + for _, tc := range []struct { name string src, dst copyKind @@ -94,19 +100,17 @@ func TestCopy(t *testing.T) { mem := memory.NewCheckedAllocator(memory.NewGoAllocator()) defer mem.AssertSize(t, 0) - f, err := ioutil.TempFile("", "arrow-ipc-") + f, err := ioutil.TempFile(tempDir, "go-arrow-copy-") if err != nil { t.Fatal(err) } defer f.Close() - defer os.Remove(f.Name()) - o, err := ioutil.TempFile("", "arrow-ipc-") + o, err := ioutil.TempFile(tempDir, "go-arrow-copy-") if err != nil { t.Fatal(err) } defer o.Close() - defer os.Remove(o.Name()) tc.src.write(t, f, mem, recs[0].Schema(), recs) tc.src.check(t, f, mem, recs[0].Schema(), recs) diff --git a/go/arrow/internal/arrjson/arrjson_test.go b/go/arrow/internal/arrjson/arrjson_test.go index 53a0cf42b7b9..6079169c4c80 100644 --- a/go/arrow/internal/arrjson/arrjson_test.go +++ b/go/arrow/internal/arrjson/arrjson_test.go @@ -28,6 +28,12 @@ import ( ) func TestReadWrite(t *testing.T) { + tempDir, err := ioutil.TempDir("", "go-arrow-read-write-") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(tempDir) + for name, recs := range arrdata.Records { t.Run(name, func(t *testing.T) { if name == "decimal128" { @@ -36,12 +42,11 @@ func TestReadWrite(t *testing.T) { mem := memory.NewCheckedAllocator(memory.NewGoAllocator()) defer mem.AssertSize(t, 0) - f, err := ioutil.TempFile("", "arrjson-") + f, err := ioutil.TempFile(tempDir, "go-arrow-read-write-") if err != nil { t.Fatal(err) } defer f.Close() - defer os.RemoveAll(f.Name()) w, err := NewWriter(f, recs[0].Schema()) if err != nil { diff --git a/go/arrow/ipc/cmd/arrow-cat/main_test.go b/go/arrow/ipc/cmd/arrow-cat/main_test.go index d5dc9c337c36..a940d8b807e3 100644 --- a/go/arrow/ipc/cmd/arrow-cat/main_test.go +++ b/go/arrow/ipc/cmd/arrow-cat/main_test.go @@ -31,6 +31,12 @@ import ( ) func TestCatStream(t *testing.T) { + tempDir, err := ioutil.TempDir("", "go-arrow-cat-stream-") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(tempDir) + for _, tc := range []struct { name string want string @@ -174,7 +180,7 @@ record 3... defer mem.AssertSize(t, 0) fname := func() string { - f, err := ioutil.TempFile("", "go-arrow-") + f, err := ioutil.TempFile(tempDir, "go-arrow-cat-stream-") if err != nil { t.Fatal(err) } @@ -202,7 +208,6 @@ record 3... return f.Name() }() - defer os.Remove(fname) f, err := os.Open(fname) if err != nil { @@ -224,6 +229,12 @@ record 3... } func TestCatFile(t *testing.T) { + tempDir, err := ioutil.TempDir("", "go-arrow-cat-file-") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(tempDir) + for _, tc := range []struct { name string want string @@ -515,7 +526,7 @@ record 3/3... defer mem.AssertSize(t, 0) fname := func() string { - f, err := ioutil.TempFile("", "go-arrow-") + f, err := ioutil.TempFile(tempDir, "go-arrow-cat-file-") if err != nil { t.Fatal(err) } @@ -556,7 +567,6 @@ record 3/3... return f.Name() }() - defer os.Remove(fname) w := new(bytes.Buffer) err := processFile(w, fname) diff --git a/go/arrow/ipc/cmd/arrow-file-to-stream/main_test.go b/go/arrow/ipc/cmd/arrow-file-to-stream/main_test.go index e32bf4810cb1..d42518a6cd7a 100644 --- a/go/arrow/ipc/cmd/arrow-file-to-stream/main_test.go +++ b/go/arrow/ipc/cmd/arrow-file-to-stream/main_test.go @@ -27,25 +27,30 @@ import ( ) func TestFileToStream(t *testing.T) { + tempDir, err := ioutil.TempDir("", "go-arrow-file-to-stream-") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(tempDir) + for name, recs := range arrdata.Records { t.Run(name, func(t *testing.T) { mem := memory.NewCheckedAllocator(memory.NewGoAllocator()) defer mem.AssertSize(t, 0) - f, err := ioutil.TempFile("", "arrow-ipc-") + f, err := ioutil.TempFile(tempDir, "go-arrow-file-to-stream-") if err != nil { t.Fatal(err) } defer f.Close() - defer os.Remove(f.Name()) arrdata.WriteFile(t, f, mem, recs[0].Schema(), recs) - o, err := ioutil.TempFile("", "arrow-ipc-") + o, err := ioutil.TempFile(tempDir, "go-arrow-file-to-stream-") if err != nil { t.Fatal(err) } - defer os.Remove(o.Name()) + defer o.Close() err = processFile(o, f.Name()) if err != nil { diff --git a/go/arrow/ipc/cmd/arrow-json-integration-test/main_test.go b/go/arrow/ipc/cmd/arrow-json-integration-test/main_test.go index 559f6db3bfb6..0d189e7ef502 100644 --- a/go/arrow/ipc/cmd/arrow-json-integration-test/main_test.go +++ b/go/arrow/ipc/cmd/arrow-json-integration-test/main_test.go @@ -26,6 +26,12 @@ import ( ) func TestIntegration(t *testing.T) { + tempDir, err := ioutil.TempDir("", "go-arrow-integration-") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(tempDir) + const verbose = true for name, recs := range arrdata.Records { t.Run(name, func(t *testing.T) { @@ -35,22 +41,20 @@ func TestIntegration(t *testing.T) { mem := memory.NewCheckedAllocator(memory.NewGoAllocator()) defer mem.AssertSize(t, 0) - af1, err := ioutil.TempFile("", "arrow-json-integration-") + af1, err := ioutil.TempFile(tempDir, "go-arrow-integration-") if err != nil { t.Fatal(err) } defer af1.Close() - defer os.RemoveAll(af1.Name()) arrdata.WriteFile(t, af1, mem, recs[0].Schema(), recs) arrdata.CheckArrowFile(t, af1, mem, recs[0].Schema(), recs) - aj, err := ioutil.TempFile("", "arrow-json-integration-") + aj, err := ioutil.TempFile(tempDir, "arrow-json-integration-") if err != nil { t.Fatal(err) } defer aj.Close() - defer os.RemoveAll(aj.Name()) err = cnvToJSON(af1.Name(), aj.Name(), verbose) if err != nil { @@ -62,12 +66,11 @@ func TestIntegration(t *testing.T) { t.Fatal(err) } - af2, err := ioutil.TempFile("", "arrow-json-integration-") + af2, err := ioutil.TempFile(tempDir, "go-arrow-integration-") if err != nil { t.Fatal(err) } - af2.Close() - os.RemoveAll(af2.Name()) + defer af2.Close() err = cnvToARROW(af2.Name(), aj.Name(), verbose) if err != nil { diff --git a/go/arrow/ipc/cmd/arrow-ls/main_test.go b/go/arrow/ipc/cmd/arrow-ls/main_test.go index 71db468ed1b0..5d6ac92649ed 100644 --- a/go/arrow/ipc/cmd/arrow-ls/main_test.go +++ b/go/arrow/ipc/cmd/arrow-ls/main_test.go @@ -31,6 +31,12 @@ import ( ) func TestLsStream(t *testing.T) { + tempDir, err := ioutil.TempDir("", "go-arrow-ls-stream-") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(tempDir) + for _, tc := range []struct { name string want string @@ -119,7 +125,7 @@ records: 3 defer mem.AssertSize(t, 0) fname := func() string { - f, err := ioutil.TempFile("", "go-arrow-") + f, err := ioutil.TempFile(tempDir, "go-arrow-ls-stream-") if err != nil { t.Fatal(err) } @@ -147,7 +153,6 @@ records: 3 return f.Name() }() - defer os.Remove(fname) f, err := os.Open(fname) if err != nil { @@ -169,6 +174,12 @@ records: 3 } func TestLsFile(t *testing.T) { + tempDir, err := ioutil.TempDir("", "go-arrow-ls-file-") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(tempDir) + for _, tc := range []struct { stream bool name string @@ -274,7 +285,7 @@ records: 3 defer mem.AssertSize(t, 0) fname := func() string { - f, err := ioutil.TempFile("", "go-arrow-") + f, err := ioutil.TempFile(tempDir, "go-arrow-ls-file-") if err != nil { t.Fatal(err) } @@ -315,7 +326,6 @@ records: 3 return f.Name() }() - defer os.Remove(fname) w := new(bytes.Buffer) err := processFile(w, fname) diff --git a/go/arrow/ipc/cmd/arrow-stream-to-file/main_test.go b/go/arrow/ipc/cmd/arrow-stream-to-file/main_test.go index fc7d995b11a8..953dfbe2b89f 100644 --- a/go/arrow/ipc/cmd/arrow-stream-to-file/main_test.go +++ b/go/arrow/ipc/cmd/arrow-stream-to-file/main_test.go @@ -27,17 +27,22 @@ import ( ) func TestStreamToFile(t *testing.T) { + tempDir, err := ioutil.TempDir("", "go-arrow-stream-to-file-") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(tempDir) + for name, recs := range arrdata.Records { t.Run(name, func(t *testing.T) { mem := memory.NewCheckedAllocator(memory.NewGoAllocator()) defer mem.AssertSize(t, 0) - f, err := ioutil.TempFile("", "arrow-ipc-") + f, err := ioutil.TempFile(tempDir, "go-arrow-stream-to-file-") if err != nil { t.Fatal(err) } defer f.Close() - defer os.Remove(f.Name()) arrdata.WriteStream(t, f, mem, recs[0].Schema(), recs) @@ -51,11 +56,10 @@ func TestStreamToFile(t *testing.T) { t.Fatal(err) } - o, err := ioutil.TempFile("", "arrow-ipc-") + o, err := ioutil.TempFile(tempDir, "go-arrow-stream-to-file-") if err != nil { t.Fatal(err) } - defer os.Remove(o.Name()) err = processStream(o, f) if err != nil { diff --git a/go/arrow/ipc/file_test.go b/go/arrow/ipc/file_test.go index 98e1bded907c..8c5d515ba5e5 100644 --- a/go/arrow/ipc/file_test.go +++ b/go/arrow/ipc/file_test.go @@ -26,17 +26,22 @@ import ( ) func TestFile(t *testing.T) { + tempDir, err := ioutil.TempDir("", "go-arrow-file-") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(tempDir) + for name, recs := range arrdata.Records { t.Run(name, func(t *testing.T) { mem := memory.NewCheckedAllocator(memory.NewGoAllocator()) defer mem.AssertSize(t, 0) - f, err := ioutil.TempFile("", "arrow-ipc-") + f, err := ioutil.TempFile(tempDir, "go-arrow-file-") if err != nil { t.Fatal(err) } defer f.Close() - defer os.Remove(f.Name()) arrdata.WriteFile(t, f, mem, recs[0].Schema(), recs) arrdata.CheckArrowFile(t, f, mem, recs[0].Schema(), recs) diff --git a/go/arrow/ipc/stream_test.go b/go/arrow/ipc/stream_test.go index 65d2c721594b..596a3f97bfde 100644 --- a/go/arrow/ipc/stream_test.go +++ b/go/arrow/ipc/stream_test.go @@ -27,17 +27,22 @@ import ( ) func TestStream(t *testing.T) { + tempDir, err := ioutil.TempDir("", "go-arrow-stream-") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(tempDir) + for name, recs := range arrdata.Records { t.Run(name, func(t *testing.T) { mem := memory.NewCheckedAllocator(memory.NewGoAllocator()) defer mem.AssertSize(t, 0) - f, err := ioutil.TempFile("", "arrow-ipc-") + f, err := ioutil.TempFile(tempDir, "go-arrow-stream-") if err != nil { t.Fatal(err) } defer f.Close() - defer os.Remove(f.Name()) arrdata.WriteStream(t, f, mem, recs[0].Schema(), recs) From a3596c9be3723031cc6f8805d6d95f977acb9f2c Mon Sep 17 00:00:00 2001 From: arw2019 Date: Fri, 7 Aug 2020 17:15:23 +0200 Subject: [PATCH 011/460] ARROW-9546: [Python] Clean up Pandas Metadata Conversion test Cosmetic improvement to a pyarrow Pandas test. I rewrote the metadata conversion test using `_check_pandas_roundtrip`. Closes #7821 from arw2019/clean-pyarrow-test_pandas Authored-by: arw2019 Signed-off-by: Joris Van den Bossche --- python/pyarrow/tests/test_pandas.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/python/pyarrow/tests/test_pandas.py b/python/pyarrow/tests/test_pandas.py index 4ded34164b87..b023e394ef9e 100644 --- a/python/pyarrow/tests/test_pandas.py +++ b/python/pyarrow/tests/test_pandas.py @@ -444,9 +444,7 @@ def test_multiindex_duplicate_values(self): df = pd.DataFrame({'numbers': numbers}, index=index) - table = pa.Table.from_pandas(df) - result_df = table.to_pandas() - tm.assert_frame_equal(result_df, df) + _check_pandas_roundtrip(df, preserve_index=True) def test_metadata_with_mixed_types(self): df = pd.DataFrame({'data': [b'some_bytes', 'some_unicode']}) From 4489cb7dfd87941bfff28a731e585a2e9d0b67be Mon Sep 17 00:00:00 2001 From: FredGan Date: Fri, 7 Aug 2020 18:33:55 +0200 Subject: [PATCH 012/460] ARROW-9462:[Go] The Indentation after the first Record in arrjson writer is incorrect Closes #7752 from fredgan/indentation-9462 Authored-by: FredGan Signed-off-by: Sebastien Binet --- go/arrow/internal/arrjson/arrjson_test.go | 2986 +++++++++++++++++++++ go/arrow/internal/arrjson/writer.go | 2 +- 2 files changed, 2987 insertions(+), 1 deletion(-) diff --git a/go/arrow/internal/arrjson/arrjson_test.go b/go/arrow/internal/arrjson/arrjson_test.go index 6079169c4c80..173db12b4013 100644 --- a/go/arrow/internal/arrjson/arrjson_test.go +++ b/go/arrow/internal/arrjson/arrjson_test.go @@ -28,6 +28,19 @@ import ( ) func TestReadWrite(t *testing.T) { + wantJSONs := make(map[string]string) + wantJSONs["nulls"] = makeNullWantJSONs() + wantJSONs["primitives"] = makePrimitiveWantJSONs() + wantJSONs["structs"] = makeStructsWantJSONs() + wantJSONs["lists"] = makeListsWantJSONs() + wantJSONs["strings"] = makeStringsWantJSONs() + wantJSONs["fixed_size_lists"] = makeFixedSizeListsWantJSONs() + wantJSONs["fixed_width_types"] = makeFixedWidthTypesWantJSONs() + wantJSONs["fixed_size_binaries"] = makeFixedSizeBinariesWantJSONs() + wantJSONs["intervals"] = makeIntervalsWantJSONs() + wantJSONs["durations"] = makeDurationsWantJSONs() + wantJSONs["decimal128"] = makeDecimal128sWantJSONs() + tempDir, err := ioutil.TempDir("", "go-arrow-read-write-") if err != nil { t.Fatal(err) @@ -71,6 +84,11 @@ func TestReadWrite(t *testing.T) { t.Fatalf("could not sync data to disk: %v", err) } + fileBytes, _ := ioutil.ReadFile(f.Name()) + if wantJSONs[name] != string(fileBytes) { + t.Fatalf("not expected JSON pretty output for case: %v", name) + } + _, err = f.Seek(0, io.SeekStart) if err != nil { t.Fatalf("could not rewind file: %v", err) @@ -116,3 +134,2971 @@ func TestReadWrite(t *testing.T) { }) } } + +func makeNullWantJSONs() string { + return `{ + "schema": { + "fields": [ + { + "name": "nulls", + "type": { + "name": "null" + }, + "nullable": true, + "children": [] + } + ] + }, + "batches": [ + { + "count": 5, + "columns": [ + { + "name": "nulls", + "count": 5 + } + ] + }, + { + "count": 5, + "columns": [ + { + "name": "nulls", + "count": 5 + } + ] + }, + { + "count": 5, + "columns": [ + { + "name": "nulls", + "count": 5 + } + ] + } + ] +}` +} + +func makePrimitiveWantJSONs() string { + return `{ + "schema": { + "fields": [ + { + "name": "bools", + "type": { + "name": "bool" + }, + "nullable": true, + "children": [] + }, + { + "name": "int8s", + "type": { + "name": "int", + "isSigned": true, + "bitWidth": 8 + }, + "nullable": true, + "children": [] + }, + { + "name": "int16s", + "type": { + "name": "int", + "isSigned": true, + "bitWidth": 16 + }, + "nullable": true, + "children": [] + }, + { + "name": "int32s", + "type": { + "name": "int", + "isSigned": true, + "bitWidth": 32 + }, + "nullable": true, + "children": [] + }, + { + "name": "int64s", + "type": { + "name": "int", + "isSigned": true, + "bitWidth": 64 + }, + "nullable": true, + "children": [] + }, + { + "name": "uint8s", + "type": { + "name": "int", + "bitWidth": 8 + }, + "nullable": true, + "children": [] + }, + { + "name": "uint16s", + "type": { + "name": "int", + "bitWidth": 16 + }, + "nullable": true, + "children": [] + }, + { + "name": "uint32s", + "type": { + "name": "int", + "bitWidth": 32 + }, + "nullable": true, + "children": [] + }, + { + "name": "uint64s", + "type": { + "name": "int", + "bitWidth": 64 + }, + "nullable": true, + "children": [] + }, + { + "name": "float32s", + "type": { + "name": "floatingpoint", + "precision": "SINGLE" + }, + "nullable": true, + "children": [] + }, + { + "name": "float64s", + "type": { + "name": "floatingpoint", + "precision": "DOUBLE" + }, + "nullable": true, + "children": [] + } + ] + }, + "batches": [ + { + "count": 5, + "columns": [ + { + "name": "bools", + "count": 5, + "VALIDITY": [ + 1, + 0, + 0, + 1, + 1 + ], + "DATA": [ + true, + false, + true, + false, + true + ] + }, + { + "name": "int8s", + "count": 5, + "VALIDITY": [ + 1, + 0, + 0, + 1, + 1 + ], + "DATA": [ + -1, + -2, + -3, + -4, + -5 + ] + }, + { + "name": "int16s", + "count": 5, + "VALIDITY": [ + 1, + 0, + 0, + 1, + 1 + ], + "DATA": [ + -1, + -2, + -3, + -4, + -5 + ] + }, + { + "name": "int32s", + "count": 5, + "VALIDITY": [ + 1, + 0, + 0, + 1, + 1 + ], + "DATA": [ + -1, + -2, + -3, + -4, + -5 + ] + }, + { + "name": "int64s", + "count": 5, + "VALIDITY": [ + 1, + 0, + 0, + 1, + 1 + ], + "DATA": [ + "-1", + "0", + "0", + "-4", + "-5" + ] + }, + { + "name": "uint8s", + "count": 5, + "VALIDITY": [ + 1, + 0, + 0, + 1, + 1 + ], + "DATA": [ + 1, + 2, + 3, + 4, + 5 + ] + }, + { + "name": "uint16s", + "count": 5, + "VALIDITY": [ + 1, + 0, + 0, + 1, + 1 + ], + "DATA": [ + 1, + 2, + 3, + 4, + 5 + ] + }, + { + "name": "uint32s", + "count": 5, + "VALIDITY": [ + 1, + 0, + 0, + 1, + 1 + ], + "DATA": [ + 1, + 2, + 3, + 4, + 5 + ] + }, + { + "name": "uint64s", + "count": 5, + "VALIDITY": [ + 1, + 0, + 0, + 1, + 1 + ], + "DATA": [ + "1", + "0", + "0", + "4", + "5" + ] + }, + { + "name": "float32s", + "count": 5, + "VALIDITY": [ + 1, + 0, + 0, + 1, + 1 + ], + "DATA": [ + 1, + 2, + 3, + 4, + 5 + ] + }, + { + "name": "float64s", + "count": 5, + "VALIDITY": [ + 1, + 0, + 0, + 1, + 1 + ], + "DATA": [ + 1, + 2, + 3, + 4, + 5 + ] + } + ] + }, + { + "count": 5, + "columns": [ + { + "name": "bools", + "count": 5, + "VALIDITY": [ + 1, + 0, + 0, + 1, + 1 + ], + "DATA": [ + true, + false, + true, + false, + true + ] + }, + { + "name": "int8s", + "count": 5, + "VALIDITY": [ + 1, + 0, + 0, + 1, + 1 + ], + "DATA": [ + -11, + -12, + -13, + -14, + -15 + ] + }, + { + "name": "int16s", + "count": 5, + "VALIDITY": [ + 1, + 0, + 0, + 1, + 1 + ], + "DATA": [ + -11, + -12, + -13, + -14, + -15 + ] + }, + { + "name": "int32s", + "count": 5, + "VALIDITY": [ + 1, + 0, + 0, + 1, + 1 + ], + "DATA": [ + -11, + -12, + -13, + -14, + -15 + ] + }, + { + "name": "int64s", + "count": 5, + "VALIDITY": [ + 1, + 0, + 0, + 1, + 1 + ], + "DATA": [ + "-11", + "0", + "0", + "-14", + "-15" + ] + }, + { + "name": "uint8s", + "count": 5, + "VALIDITY": [ + 1, + 0, + 0, + 1, + 1 + ], + "DATA": [ + 11, + 12, + 13, + 14, + 15 + ] + }, + { + "name": "uint16s", + "count": 5, + "VALIDITY": [ + 1, + 0, + 0, + 1, + 1 + ], + "DATA": [ + 11, + 12, + 13, + 14, + 15 + ] + }, + { + "name": "uint32s", + "count": 5, + "VALIDITY": [ + 1, + 0, + 0, + 1, + 1 + ], + "DATA": [ + 11, + 12, + 13, + 14, + 15 + ] + }, + { + "name": "uint64s", + "count": 5, + "VALIDITY": [ + 1, + 0, + 0, + 1, + 1 + ], + "DATA": [ + "11", + "0", + "0", + "14", + "15" + ] + }, + { + "name": "float32s", + "count": 5, + "VALIDITY": [ + 1, + 0, + 0, + 1, + 1 + ], + "DATA": [ + 11, + 12, + 13, + 14, + 15 + ] + }, + { + "name": "float64s", + "count": 5, + "VALIDITY": [ + 1, + 0, + 0, + 1, + 1 + ], + "DATA": [ + 11, + 12, + 13, + 14, + 15 + ] + } + ] + }, + { + "count": 5, + "columns": [ + { + "name": "bools", + "count": 5, + "VALIDITY": [ + 1, + 0, + 0, + 1, + 1 + ], + "DATA": [ + true, + false, + true, + false, + true + ] + }, + { + "name": "int8s", + "count": 5, + "VALIDITY": [ + 1, + 0, + 0, + 1, + 1 + ], + "DATA": [ + -21, + -22, + -23, + -24, + -25 + ] + }, + { + "name": "int16s", + "count": 5, + "VALIDITY": [ + 1, + 0, + 0, + 1, + 1 + ], + "DATA": [ + -21, + -22, + -23, + -24, + -25 + ] + }, + { + "name": "int32s", + "count": 5, + "VALIDITY": [ + 1, + 0, + 0, + 1, + 1 + ], + "DATA": [ + -21, + -22, + -23, + -24, + -25 + ] + }, + { + "name": "int64s", + "count": 5, + "VALIDITY": [ + 1, + 0, + 0, + 1, + 1 + ], + "DATA": [ + "-21", + "0", + "0", + "-24", + "-25" + ] + }, + { + "name": "uint8s", + "count": 5, + "VALIDITY": [ + 1, + 0, + 0, + 1, + 1 + ], + "DATA": [ + 21, + 22, + 23, + 24, + 25 + ] + }, + { + "name": "uint16s", + "count": 5, + "VALIDITY": [ + 1, + 0, + 0, + 1, + 1 + ], + "DATA": [ + 21, + 22, + 23, + 24, + 25 + ] + }, + { + "name": "uint32s", + "count": 5, + "VALIDITY": [ + 1, + 0, + 0, + 1, + 1 + ], + "DATA": [ + 21, + 22, + 23, + 24, + 25 + ] + }, + { + "name": "uint64s", + "count": 5, + "VALIDITY": [ + 1, + 0, + 0, + 1, + 1 + ], + "DATA": [ + "21", + "0", + "0", + "24", + "25" + ] + }, + { + "name": "float32s", + "count": 5, + "VALIDITY": [ + 1, + 0, + 0, + 1, + 1 + ], + "DATA": [ + 21, + 22, + 23, + 24, + 25 + ] + }, + { + "name": "float64s", + "count": 5, + "VALIDITY": [ + 1, + 0, + 0, + 1, + 1 + ], + "DATA": [ + 21, + 22, + 23, + 24, + 25 + ] + } + ] + } + ] +}` +} + +func makeStructsWantJSONs() string { + return `{ + "schema": { + "fields": [ + { + "name": "struct_nullable", + "type": { + "name": "struct" + }, + "nullable": true, + "children": [ + { + "name": "f1", + "type": { + "name": "int", + "isSigned": true, + "bitWidth": 32 + }, + "nullable": false, + "children": [] + }, + { + "name": "f2", + "type": { + "name": "utf8" + }, + "nullable": false, + "children": [] + } + ] + } + ] + }, + "batches": [ + { + "count": 25, + "columns": [ + { + "name": "struct_nullable", + "count": 25, + "VALIDITY": [ + 1, + 0, + 1, + 1, + 1, + 1, + 0, + 1, + 1, + 1, + 1, + 0, + 1, + 1, + 1, + 1, + 0, + 1, + 1, + 1, + 1, + 0, + 1, + 1, + 1 + ], + "children": [ + { + "name": "f1", + "count": 25, + "VALIDITY": [ + 1, + 0, + 0, + 1, + 1, + 1, + 0, + 0, + 1, + 1, + 1, + 0, + 0, + 1, + 1, + 1, + 0, + 0, + 1, + 1, + 1, + 0, + 0, + 1, + 1 + ], + "DATA": [ + -1, + 0, + 0, + -4, + -5, + -11, + 0, + 0, + -14, + -15, + -21, + 0, + 0, + -24, + -25, + -31, + 0, + 0, + -34, + -35, + -41, + 0, + 0, + -44, + -45 + ] + }, + { + "name": "f2", + "count": 25, + "VALIDITY": [ + 1, + 0, + 0, + 1, + 1, + 1, + 0, + 0, + 1, + 1, + 1, + 0, + 0, + 1, + 1, + 1, + 0, + 0, + 1, + 1, + 1, + 0, + 0, + 1, + 1 + ], + "DATA": [ + "111", + "", + "", + "444", + "555", + "1111", + "", + "", + "1444", + "1555", + "2111", + "", + "", + "2444", + "2555", + "3111", + "", + "", + "3444", + "3555", + "4111", + "", + "", + "4444", + "4555" + ] + } + ] + } + ] + }, + { + "count": 25, + "columns": [ + { + "name": "struct_nullable", + "count": 25, + "VALIDITY": [ + 1, + 0, + 0, + 1, + 1, + 1, + 0, + 0, + 1, + 1, + 1, + 0, + 0, + 1, + 1, + 1, + 0, + 0, + 1, + 1, + 1, + 0, + 0, + 1, + 1 + ], + "children": [ + { + "name": "f1", + "count": 25, + "VALIDITY": [ + 1, + 0, + 0, + 1, + 1, + 1, + 0, + 0, + 1, + 1, + 1, + 0, + 0, + 1, + 1, + 1, + 0, + 0, + 1, + 1, + 1, + 0, + 0, + 1, + 1 + ], + "DATA": [ + 1, + 0, + 0, + 4, + 5, + 11, + 0, + 0, + 14, + 15, + 21, + 0, + 0, + 24, + 25, + 31, + 0, + 0, + 34, + 35, + 41, + 0, + 0, + 44, + 45 + ] + }, + { + "name": "f2", + "count": 25, + "VALIDITY": [ + 1, + 0, + 0, + 1, + 1, + 1, + 0, + 0, + 1, + 1, + 1, + 0, + 0, + 1, + 1, + 1, + 0, + 0, + 1, + 1, + 1, + 0, + 0, + 1, + 1 + ], + "DATA": [ + "-111", + "", + "", + "-444", + "-555", + "-1111", + "", + "", + "-1444", + "-1555", + "-2111", + "", + "", + "-2444", + "-2555", + "-3111", + "", + "", + "-3444", + "-3555", + "-4111", + "", + "", + "-4444", + "-4555" + ] + } + ] + } + ] + } + ] +}` +} + +func makeListsWantJSONs() string { + return `{ + "schema": { + "fields": [ + { + "name": "list_nullable", + "type": { + "name": "list" + }, + "nullable": true, + "children": [ + { + "name": "item", + "type": { + "name": "int", + "isSigned": true, + "bitWidth": 32 + }, + "nullable": true, + "children": [] + } + ] + } + ] + }, + "batches": [ + { + "count": 3, + "columns": [ + { + "name": "list_nullable", + "count": 3, + "VALIDITY": [ + 1, + 1, + 1 + ], + "OFFSET": [ + 0, + 5, + 10, + 15 + ], + "children": [ + { + "name": "item", + "count": 15, + "VALIDITY": [ + 1, + 0, + 0, + 1, + 1, + 1, + 0, + 0, + 1, + 1, + 1, + 0, + 0, + 1, + 1 + ], + "DATA": [ + 1, + 0, + 0, + 4, + 5, + 11, + 0, + 0, + 14, + 15, + 21, + 0, + 0, + 24, + 25 + ] + } + ] + } + ] + }, + { + "count": 3, + "columns": [ + { + "name": "list_nullable", + "count": 3, + "VALIDITY": [ + 1, + 1, + 1 + ], + "OFFSET": [ + 0, + 5, + 10, + 15 + ], + "children": [ + { + "name": "item", + "count": 15, + "VALIDITY": [ + 1, + 0, + 0, + 1, + 1, + 1, + 0, + 0, + 1, + 1, + 1, + 0, + 0, + 1, + 1 + ], + "DATA": [ + -1, + 0, + 0, + -4, + -5, + -11, + 0, + 0, + -14, + -15, + -21, + 0, + 0, + -24, + -25 + ] + } + ] + } + ] + }, + { + "count": 3, + "columns": [ + { + "name": "list_nullable", + "count": 3, + "VALIDITY": [ + 1, + 0, + 1 + ], + "OFFSET": [ + 0, + 5, + 10, + 15 + ], + "children": [ + { + "name": "item", + "count": 15, + "VALIDITY": [ + 1, + 0, + 0, + 1, + 1, + 1, + 0, + 0, + 1, + 1, + 1, + 0, + 0, + 1, + 1 + ], + "DATA": [ + -1, + 0, + 0, + -4, + -5, + -11, + 0, + 0, + -14, + -15, + -21, + 0, + 0, + -24, + -25 + ] + } + ] + } + ] + }, + { + "count": 0, + "columns": [ + { + "name": "list_nullable", + "count": 0, + "OFFSET": [ + 0 + ], + "children": [ + { + "name": "item", + "count": 0 + } + ] + } + ] + } + ] +}` +} + +func makeFixedSizeListsWantJSONs() string { + return `{ + "schema": { + "fields": [ + { + "name": "fixed_size_list_nullable", + "type": { + "name": "fixedsizelist", + "listSize": 3 + }, + "nullable": true, + "children": [ + { + "name": "item", + "type": { + "name": "int", + "isSigned": true, + "bitWidth": 32 + }, + "nullable": true, + "children": [] + } + ] + } + ] + }, + "batches": [ + { + "count": 3, + "columns": [ + { + "name": "fixed_size_list_nullable", + "count": 3, + "VALIDITY": [ + 1, + 1, + 1 + ], + "children": [ + { + "name": "", + "count": 9, + "VALIDITY": [ + 1, + 0, + 1, + 1, + 0, + 1, + 1, + 0, + 1 + ], + "DATA": [ + 1, + 0, + 3, + 11, + 0, + 13, + 21, + 0, + 23 + ] + } + ] + } + ] + }, + { + "count": 3, + "columns": [ + { + "name": "fixed_size_list_nullable", + "count": 3, + "VALIDITY": [ + 1, + 1, + 1 + ], + "children": [ + { + "name": "", + "count": 9, + "VALIDITY": [ + 1, + 0, + 1, + 1, + 0, + 1, + 1, + 0, + 1 + ], + "DATA": [ + -1, + 0, + -3, + -11, + 0, + -13, + -21, + 0, + -23 + ] + } + ] + } + ] + }, + { + "count": 3, + "columns": [ + { + "name": "fixed_size_list_nullable", + "count": 3, + "VALIDITY": [ + 1, + 0, + 1 + ], + "children": [ + { + "name": "", + "count": 9, + "VALIDITY": [ + 1, + 0, + 1, + 1, + 0, + 1, + 1, + 0, + 1 + ], + "DATA": [ + -1, + 0, + -3, + -11, + 0, + -13, + -21, + 0, + -23 + ] + } + ] + } + ] + } + ] +}` +} + +func makeStringsWantJSONs() string { + return `{ + "schema": { + "fields": [ + { + "name": "strings", + "type": { + "name": "utf8" + }, + "nullable": false, + "children": [] + }, + { + "name": "bytes", + "type": { + "name": "binary" + }, + "nullable": false, + "children": [] + } + ] + }, + "batches": [ + { + "count": 5, + "columns": [ + { + "name": "strings", + "count": 5, + "VALIDITY": [ + 1, + 0, + 0, + 1, + 1 + ], + "DATA": [ + "1é", + "2", + "3", + "4", + "5" + ] + }, + { + "name": "bytes", + "count": 5, + "VALIDITY": [ + 1, + 0, + 0, + 1, + 1 + ], + "DATA": [ + "31C3A9", + "32", + "33", + "34", + "35" + ], + "OFFSET": [ + 0, + 3, + 4, + 5, + 6, + 7 + ] + } + ] + }, + { + "count": 5, + "columns": [ + { + "name": "strings", + "count": 5, + "VALIDITY": [ + 1, + 0, + 0, + 1, + 1 + ], + "DATA": [ + "11", + "22", + "33", + "44", + "55" + ] + }, + { + "name": "bytes", + "count": 5, + "VALIDITY": [ + 1, + 0, + 0, + 1, + 1 + ], + "DATA": [ + "3131", + "3232", + "3333", + "3434", + "3535" + ], + "OFFSET": [ + 0, + 2, + 4, + 6, + 8, + 10 + ] + } + ] + }, + { + "count": 5, + "columns": [ + { + "name": "strings", + "count": 5, + "VALIDITY": [ + 1, + 0, + 0, + 1, + 1 + ], + "DATA": [ + "111", + "222", + "333", + "444", + "555" + ] + }, + { + "name": "bytes", + "count": 5, + "VALIDITY": [ + 1, + 0, + 0, + 1, + 1 + ], + "DATA": [ + "313131", + "323232", + "333333", + "343434", + "353535" + ], + "OFFSET": [ + 0, + 3, + 6, + 9, + 12, + 15 + ] + } + ] + } + ] +}` +} + +func makeFixedWidthTypesWantJSONs() string { + return `{ + "schema": { + "fields": [ + { + "name": "float16s", + "type": { + "name": "floatingpoint", + "precision": "HALF" + }, + "nullable": true, + "children": [] + }, + { + "name": "time32ms", + "type": { + "name": "time", + "bitWidth": 32, + "unit": "MILLISECOND" + }, + "nullable": true, + "children": [] + }, + { + "name": "time32s", + "type": { + "name": "time", + "bitWidth": 32, + "unit": "SECOND" + }, + "nullable": true, + "children": [] + }, + { + "name": "time64ns", + "type": { + "name": "time", + "bitWidth": 64, + "unit": "NANOSECOND" + }, + "nullable": true, + "children": [] + }, + { + "name": "time64us", + "type": { + "name": "time", + "bitWidth": 64, + "unit": "MICROSECOND" + }, + "nullable": true, + "children": [] + }, + { + "name": "timestamp_s", + "type": { + "name": "timestamp", + "unit": "SECOND", + "timezone": "UTC" + }, + "nullable": true, + "children": [] + }, + { + "name": "timestamp_ms", + "type": { + "name": "timestamp", + "unit": "MILLISECOND", + "timezone": "UTC" + }, + "nullable": true, + "children": [] + }, + { + "name": "timestamp_us", + "type": { + "name": "timestamp", + "unit": "MICROSECOND", + "timezone": "UTC" + }, + "nullable": true, + "children": [] + }, + { + "name": "timestamp_ns", + "type": { + "name": "timestamp", + "unit": "NANOSECOND", + "timezone": "UTC" + }, + "nullable": true, + "children": [] + }, + { + "name": "date32s", + "type": { + "name": "date", + "unit": "DAY" + }, + "nullable": true, + "children": [] + }, + { + "name": "date64s", + "type": { + "name": "date", + "unit": "MILLISECOND" + }, + "nullable": true, + "children": [] + } + ] + }, + "batches": [ + { + "count": 5, + "columns": [ + { + "name": "float16s", + "count": 5, + "VALIDITY": [ + 1, + 0, + 0, + 1, + 1 + ], + "DATA": [ + 1, + 2, + 3, + 4, + 5 + ] + }, + { + "name": "time32ms", + "count": 5, + "VALIDITY": [ + 1, + 0, + 0, + 1, + 1 + ], + "DATA": [ + -2, + -1, + 0, + 1, + 2 + ] + }, + { + "name": "time32s", + "count": 5, + "VALIDITY": [ + 1, + 0, + 0, + 1, + 1 + ], + "DATA": [ + -2, + -1, + 0, + 1, + 2 + ] + }, + { + "name": "time64ns", + "count": 5, + "VALIDITY": [ + 1, + 0, + 0, + 1, + 1 + ], + "DATA": [ + "-2", + "0", + "0", + "1", + "2" + ] + }, + { + "name": "time64us", + "count": 5, + "VALIDITY": [ + 1, + 0, + 0, + 1, + 1 + ], + "DATA": [ + "-2", + "0", + "0", + "1", + "2" + ] + }, + { + "name": "timestamp_s", + "count": 5, + "VALIDITY": [ + 1, + 0, + 0, + 1, + 1 + ], + "DATA": [ + "0", + "0", + "0", + "3", + "4" + ] + }, + { + "name": "timestamp_ms", + "count": 5, + "VALIDITY": [ + 1, + 0, + 0, + 1, + 1 + ], + "DATA": [ + "0", + "0", + "0", + "3", + "4" + ] + }, + { + "name": "timestamp_us", + "count": 5, + "VALIDITY": [ + 1, + 0, + 0, + 1, + 1 + ], + "DATA": [ + "0", + "0", + "0", + "3", + "4" + ] + }, + { + "name": "timestamp_ns", + "count": 5, + "VALIDITY": [ + 1, + 0, + 0, + 1, + 1 + ], + "DATA": [ + "0", + "0", + "0", + "3", + "4" + ] + }, + { + "name": "date32s", + "count": 5, + "VALIDITY": [ + 1, + 0, + 0, + 1, + 1 + ], + "DATA": [ + -2, + -1, + 0, + 1, + 2 + ] + }, + { + "name": "date64s", + "count": 5, + "VALIDITY": [ + 1, + 0, + 0, + 1, + 1 + ], + "DATA": [ + "-2", + "0", + "0", + "1", + "2" + ] + } + ] + }, + { + "count": 5, + "columns": [ + { + "name": "float16s", + "count": 5, + "VALIDITY": [ + 1, + 0, + 0, + 1, + 1 + ], + "DATA": [ + 11, + 12, + 13, + 14, + 15 + ] + }, + { + "name": "time32ms", + "count": 5, + "VALIDITY": [ + 1, + 0, + 0, + 1, + 1 + ], + "DATA": [ + -12, + -11, + 10, + 11, + 12 + ] + }, + { + "name": "time32s", + "count": 5, + "VALIDITY": [ + 1, + 0, + 0, + 1, + 1 + ], + "DATA": [ + -12, + -11, + 10, + 11, + 12 + ] + }, + { + "name": "time64ns", + "count": 5, + "VALIDITY": [ + 1, + 0, + 0, + 1, + 1 + ], + "DATA": [ + "-12", + "0", + "0", + "11", + "12" + ] + }, + { + "name": "time64us", + "count": 5, + "VALIDITY": [ + 1, + 0, + 0, + 1, + 1 + ], + "DATA": [ + "-12", + "0", + "0", + "11", + "12" + ] + }, + { + "name": "timestamp_s", + "count": 5, + "VALIDITY": [ + 1, + 0, + 0, + 1, + 1 + ], + "DATA": [ + "10", + "0", + "0", + "13", + "14" + ] + }, + { + "name": "timestamp_ms", + "count": 5, + "VALIDITY": [ + 1, + 0, + 0, + 1, + 1 + ], + "DATA": [ + "10", + "0", + "0", + "13", + "14" + ] + }, + { + "name": "timestamp_us", + "count": 5, + "VALIDITY": [ + 1, + 0, + 0, + 1, + 1 + ], + "DATA": [ + "10", + "0", + "0", + "13", + "14" + ] + }, + { + "name": "timestamp_ns", + "count": 5, + "VALIDITY": [ + 1, + 0, + 0, + 1, + 1 + ], + "DATA": [ + "10", + "0", + "0", + "13", + "14" + ] + }, + { + "name": "date32s", + "count": 5, + "VALIDITY": [ + 1, + 0, + 0, + 1, + 1 + ], + "DATA": [ + -12, + -11, + 10, + 11, + 12 + ] + }, + { + "name": "date64s", + "count": 5, + "VALIDITY": [ + 1, + 0, + 0, + 1, + 1 + ], + "DATA": [ + "-12", + "0", + "0", + "11", + "12" + ] + } + ] + }, + { + "count": 5, + "columns": [ + { + "name": "float16s", + "count": 5, + "VALIDITY": [ + 1, + 0, + 0, + 1, + 1 + ], + "DATA": [ + 21, + 22, + 23, + 24, + 25 + ] + }, + { + "name": "time32ms", + "count": 5, + "VALIDITY": [ + 1, + 0, + 0, + 1, + 1 + ], + "DATA": [ + -22, + -21, + 20, + 21, + 22 + ] + }, + { + "name": "time32s", + "count": 5, + "VALIDITY": [ + 1, + 0, + 0, + 1, + 1 + ], + "DATA": [ + -22, + -21, + 20, + 21, + 22 + ] + }, + { + "name": "time64ns", + "count": 5, + "VALIDITY": [ + 1, + 0, + 0, + 1, + 1 + ], + "DATA": [ + "-22", + "0", + "0", + "21", + "22" + ] + }, + { + "name": "time64us", + "count": 5, + "VALIDITY": [ + 1, + 0, + 0, + 1, + 1 + ], + "DATA": [ + "-22", + "0", + "0", + "21", + "22" + ] + }, + { + "name": "timestamp_s", + "count": 5, + "VALIDITY": [ + 1, + 0, + 0, + 1, + 1 + ], + "DATA": [ + "20", + "0", + "0", + "23", + "24" + ] + }, + { + "name": "timestamp_ms", + "count": 5, + "VALIDITY": [ + 1, + 0, + 0, + 1, + 1 + ], + "DATA": [ + "20", + "0", + "0", + "23", + "24" + ] + }, + { + "name": "timestamp_us", + "count": 5, + "VALIDITY": [ + 1, + 0, + 0, + 1, + 1 + ], + "DATA": [ + "20", + "0", + "0", + "23", + "24" + ] + }, + { + "name": "timestamp_ns", + "count": 5, + "VALIDITY": [ + 1, + 0, + 0, + 1, + 1 + ], + "DATA": [ + "20", + "0", + "0", + "23", + "24" + ] + }, + { + "name": "date32s", + "count": 5, + "VALIDITY": [ + 1, + 0, + 0, + 1, + 1 + ], + "DATA": [ + -22, + -21, + 20, + 21, + 22 + ] + }, + { + "name": "date64s", + "count": 5, + "VALIDITY": [ + 1, + 0, + 0, + 1, + 1 + ], + "DATA": [ + "-22", + "0", + "0", + "21", + "22" + ] + } + ] + } + ] +}` +} + +func makeFixedSizeBinariesWantJSONs() string { + return `{ + "schema": { + "fields": [ + { + "name": "fixed_size_binary_3", + "type": { + "name": "fixedsizebinary", + "byteWidth": 3 + }, + "nullable": true, + "children": [] + } + ] + }, + "batches": [ + { + "count": 5, + "columns": [ + { + "name": "fixed_size_binary_3", + "count": 5, + "VALIDITY": [ + 1, + 0, + 0, + 1, + 1 + ], + "DATA": [ + "303031", + "303032", + "303033", + "303034", + "303035" + ] + } + ] + }, + { + "count": 5, + "columns": [ + { + "name": "fixed_size_binary_3", + "count": 5, + "VALIDITY": [ + 1, + 0, + 0, + 1, + 1 + ], + "DATA": [ + "303131", + "303132", + "303133", + "303134", + "303135" + ] + } + ] + }, + { + "count": 5, + "columns": [ + { + "name": "fixed_size_binary_3", + "count": 5, + "VALIDITY": [ + 1, + 0, + 0, + 1, + 1 + ], + "DATA": [ + "303231", + "303232", + "303233", + "303234", + "303235" + ] + } + ] + } + ] +}` +} + +func makeIntervalsWantJSONs() string { + return `{ + "schema": { + "fields": [ + { + "name": "months", + "type": { + "name": "interval", + "unit": "YEAR_MONTH" + }, + "nullable": true, + "children": [] + }, + { + "name": "days", + "type": { + "name": "interval", + "unit": "DAY_TIME" + }, + "nullable": true, + "children": [] + } + ] + }, + "batches": [ + { + "count": 5, + "columns": [ + { + "name": "months", + "count": 5, + "VALIDITY": [ + 1, + 0, + 0, + 1, + 1 + ], + "DATA": [ + 1, + 2, + 3, + 4, + 5 + ] + }, + { + "name": "days", + "count": 5, + "VALIDITY": [ + 1, + 0, + 0, + 1, + 1 + ], + "DATA": [ + { + "days": 1, + "milliseconds": 1 + }, + { + "days": 2, + "milliseconds": 2 + }, + { + "days": 3, + "milliseconds": 3 + }, + { + "days": 4, + "milliseconds": 4 + }, + { + "days": 5, + "milliseconds": 5 + } + ] + } + ] + }, + { + "count": 5, + "columns": [ + { + "name": "months", + "count": 5, + "VALIDITY": [ + 1, + 0, + 0, + 1, + 1 + ], + "DATA": [ + 11, + 12, + 13, + 14, + 15 + ] + }, + { + "name": "days", + "count": 5, + "VALIDITY": [ + 1, + 0, + 0, + 1, + 1 + ], + "DATA": [ + { + "days": 11, + "milliseconds": 11 + }, + { + "days": 12, + "milliseconds": 12 + }, + { + "days": 13, + "milliseconds": 13 + }, + { + "days": 14, + "milliseconds": 14 + }, + { + "days": 15, + "milliseconds": 15 + } + ] + } + ] + }, + { + "count": 5, + "columns": [ + { + "name": "months", + "count": 5, + "VALIDITY": [ + 1, + 0, + 0, + 1, + 1 + ], + "DATA": [ + 21, + 22, + 23, + 24, + 25 + ] + }, + { + "name": "days", + "count": 5, + "VALIDITY": [ + 1, + 0, + 0, + 1, + 1 + ], + "DATA": [ + { + "days": 21, + "milliseconds": 21 + }, + { + "days": 22, + "milliseconds": 22 + }, + { + "days": 23, + "milliseconds": 23 + }, + { + "days": 24, + "milliseconds": 24 + }, + { + "days": 25, + "milliseconds": 25 + } + ] + } + ] + } + ] +}` +} + +func makeDurationsWantJSONs() string { + return `{ + "schema": { + "fields": [ + { + "name": "durations-s", + "type": { + "name": "duration", + "unit": "SECOND" + }, + "nullable": true, + "children": [] + }, + { + "name": "durations-ms", + "type": { + "name": "duration", + "unit": "MILLISECOND" + }, + "nullable": true, + "children": [] + }, + { + "name": "durations-us", + "type": { + "name": "duration", + "unit": "MICROSECOND" + }, + "nullable": true, + "children": [] + }, + { + "name": "durations-ns", + "type": { + "name": "duration", + "unit": "NANOSECOND" + }, + "nullable": true, + "children": [] + } + ] + }, + "batches": [ + { + "count": 5, + "columns": [ + { + "name": "durations-s", + "count": 5, + "VALIDITY": [ + 1, + 0, + 0, + 1, + 1 + ], + "DATA": [ + "1", + "0", + "0", + "4", + "5" + ] + }, + { + "name": "durations-ms", + "count": 5, + "VALIDITY": [ + 1, + 0, + 0, + 1, + 1 + ], + "DATA": [ + "1", + "0", + "0", + "4", + "5" + ] + }, + { + "name": "durations-us", + "count": 5, + "VALIDITY": [ + 1, + 0, + 0, + 1, + 1 + ], + "DATA": [ + "1", + "0", + "0", + "4", + "5" + ] + }, + { + "name": "durations-ns", + "count": 5, + "VALIDITY": [ + 1, + 0, + 0, + 1, + 1 + ], + "DATA": [ + "1", + "0", + "0", + "4", + "5" + ] + } + ] + }, + { + "count": 5, + "columns": [ + { + "name": "durations-s", + "count": 5, + "VALIDITY": [ + 1, + 0, + 0, + 1, + 1 + ], + "DATA": [ + "11", + "0", + "0", + "14", + "15" + ] + }, + { + "name": "durations-ms", + "count": 5, + "VALIDITY": [ + 1, + 0, + 0, + 1, + 1 + ], + "DATA": [ + "11", + "0", + "0", + "14", + "15" + ] + }, + { + "name": "durations-us", + "count": 5, + "VALIDITY": [ + 1, + 0, + 0, + 1, + 1 + ], + "DATA": [ + "11", + "0", + "0", + "14", + "15" + ] + }, + { + "name": "durations-ns", + "count": 5, + "VALIDITY": [ + 1, + 0, + 0, + 1, + 1 + ], + "DATA": [ + "11", + "0", + "0", + "14", + "15" + ] + } + ] + }, + { + "count": 5, + "columns": [ + { + "name": "durations-s", + "count": 5, + "VALIDITY": [ + 1, + 0, + 0, + 1, + 1 + ], + "DATA": [ + "21", + "0", + "0", + "24", + "25" + ] + }, + { + "name": "durations-ms", + "count": 5, + "VALIDITY": [ + 1, + 0, + 0, + 1, + 1 + ], + "DATA": [ + "21", + "0", + "0", + "24", + "25" + ] + }, + { + "name": "durations-us", + "count": 5, + "VALIDITY": [ + 1, + 0, + 0, + 1, + 1 + ], + "DATA": [ + "21", + "0", + "0", + "24", + "25" + ] + }, + { + "name": "durations-ns", + "count": 5, + "VALIDITY": [ + 1, + 0, + 0, + 1, + 1 + ], + "DATA": [ + "21", + "0", + "0", + "24", + "25" + ] + } + ] + } + ] +}` +} + +func makeDecimal128sWantJSONs() string { + return `` // FIXME(fredgan): implement full decimal128 JSON support +} \ No newline at end of file diff --git a/go/arrow/internal/arrjson/writer.go b/go/arrow/internal/arrjson/writer.go index ab521f74aaca..72fab46bc48d 100644 --- a/go/arrow/internal/arrjson/writer.go +++ b/go/arrow/internal/arrjson/writer.go @@ -63,7 +63,7 @@ func (w *Writer) Write(rec array.Record) error { return err } case w.nrecs > 0: - _, err := w.w.Write([]byte(",\n")) + _, err := w.w.Write([]byte(",\n" + jsonRecPrefix)) if err != nil { return err } From 9c04867930eae5454dbb1ea4c7bd869b12fc6e9d Mon Sep 17 00:00:00 2001 From: Frank Du Date: Fri, 7 Aug 2020 11:42:51 -0500 Subject: [PATCH 013/460] ARROW-9643: [C++] Only register the SIMD variants when it's supported. Fix illegal instruction on GCC 7.5 build, compiler may use advanced instruction just for a register routine. Signed-off-by: Frank Du Closes #7903 from jianxind/ARROW-9643 Authored-by: Frank Du Signed-off-by: Wes McKinney --- .../arrow/compute/kernels/aggregate_basic.cc | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/cpp/src/arrow/compute/kernels/aggregate_basic.cc b/cpp/src/arrow/compute/kernels/aggregate_basic.cc index 2f3cdda3daa4..7e74f4c8a0f4 100644 --- a/cpp/src/arrow/compute/kernels/aggregate_basic.cc +++ b/cpp/src/arrow/compute/kernels/aggregate_basic.cc @@ -21,6 +21,7 @@ #include "arrow/compute/kernels/aggregate_basic_internal.h" #include "arrow/compute/kernels/aggregate_internal.h" #include "arrow/compute/kernels/common.h" +#include "arrow/util/cpu_info.h" #include "arrow/util/make_unique.h" namespace arrow { @@ -396,11 +397,16 @@ void RegisterScalarAggregateBasic(FunctionRegistry* registry) { aggregate::AddBasicAggKernels(aggregate::SumInit, FloatingPointTypes(), float64(), func.get()); // Add the SIMD variants for sum + auto cpu_info = arrow::internal::CpuInfo::GetInstance(); #if defined(ARROW_HAVE_RUNTIME_AVX2) - aggregate::AddSumAvx2AggKernels(func.get()); + if (cpu_info->IsSupported(arrow::internal::CpuInfo::AVX2)) { + aggregate::AddSumAvx2AggKernels(func.get()); + } #endif #if defined(ARROW_HAVE_RUNTIME_AVX512) - aggregate::AddSumAvx512AggKernels(func.get()); + if (cpu_info->IsSupported(arrow::internal::CpuInfo::AVX512)) { + aggregate::AddSumAvx512AggKernels(func.get()); + } #endif DCHECK_OK(registry->AddFunction(std::move(func))); @@ -410,10 +416,14 @@ void RegisterScalarAggregateBasic(FunctionRegistry* registry) { func.get()); // Add the SIMD variants for mean #if defined(ARROW_HAVE_RUNTIME_AVX2) - aggregate::AddMeanAvx2AggKernels(func.get()); + if (cpu_info->IsSupported(arrow::internal::CpuInfo::AVX2)) { + aggregate::AddMeanAvx2AggKernels(func.get()); + } #endif #if defined(ARROW_HAVE_RUNTIME_AVX512) - aggregate::AddMeanAvx512AggKernels(func.get()); + if (cpu_info->IsSupported(arrow::internal::CpuInfo::AVX512)) { + aggregate::AddMeanAvx512AggKernels(func.get()); + } #endif DCHECK_OK(registry->AddFunction(std::move(func))); From f40e2871919b577995a8e9e2419af2e494f574cb Mon Sep 17 00:00:00 2001 From: offthewall123 Date: Fri, 7 Aug 2020 19:51:43 -0700 Subject: [PATCH 014/460] ARROW-9536: [Java] Miss parameters in PlasmaOutOfMemoryException.java Miss parameters in PlasmaOutOfMemoryException.java Closes #7815 from offthewall123/miss_parameter_bug_fix Authored-by: offthewall123 Signed-off-by: Micah Kornfield --- .../exceptions/PlasmaOutOfMemoryException.java | 8 ++++++++ .../apache/arrow/plasma/PlasmaClientTest.java | 17 ++++++++++++++++- 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/java/plasma/src/main/java/org/apache/arrow/plasma/exceptions/PlasmaOutOfMemoryException.java b/java/plasma/src/main/java/org/apache/arrow/plasma/exceptions/PlasmaOutOfMemoryException.java index fd323fa70c99..ffc4177ebbe3 100644 --- a/java/plasma/src/main/java/org/apache/arrow/plasma/exceptions/PlasmaOutOfMemoryException.java +++ b/java/plasma/src/main/java/org/apache/arrow/plasma/exceptions/PlasmaOutOfMemoryException.java @@ -22,6 +22,14 @@ */ public class PlasmaOutOfMemoryException extends RuntimeException { + public PlasmaOutOfMemoryException(String message) { + super("The plasma store ran out of memory." + message); + } + + public PlasmaOutOfMemoryException(String message, Throwable t) { + super("The plasma store ran out of memory." + message, t); + } + public PlasmaOutOfMemoryException() { super("The plasma store ran out of memory."); } diff --git a/java/plasma/src/test/java/org/apache/arrow/plasma/PlasmaClientTest.java b/java/plasma/src/test/java/org/apache/arrow/plasma/PlasmaClientTest.java index d3aebea43c47..e02ee51f9ed0 100644 --- a/java/plasma/src/test/java/org/apache/arrow/plasma/PlasmaClientTest.java +++ b/java/plasma/src/test/java/org/apache/arrow/plasma/PlasmaClientTest.java @@ -26,6 +26,7 @@ import org.apache.arrow.plasma.exceptions.DuplicateObjectException; import org.apache.arrow.plasma.exceptions.PlasmaClientException; +import org.apache.arrow.plasma.exceptions.PlasmaOutOfMemoryException; import org.junit.Assert; public class PlasmaClientTest { @@ -277,6 +278,20 @@ public void doByteBufferTest() { client.release(id); } + public void doPlasmaOutOfMemoryExceptionTest() { + System.out.println("Start PlasmaOutOfMemoryException test."); + PlasmaClient client = (PlasmaClient) pLink; + byte[] objectId = new byte[20]; + Arrays.fill(objectId, (byte) 1); + try { + ByteBuffer byteBuffer = client.create(objectId, 200000000, null); + Assert.fail("Fail to create an object, The plasma store ran out of memory."); + } catch (PlasmaOutOfMemoryException e) { + System.out.println(String.format("Expected PlasmaOutOfMemoryException: %s", e)); + System.out.println("PlasmaOutOfMemoryException test success."); + } + } + private byte[] getArrayFilledWithValue(int arrayLength, byte val) { byte[] arr = new byte[arrayLength]; Arrays.fill(arr, val); @@ -290,9 +305,9 @@ public String getStoreAddress() { public static void main(String[] args) throws Exception { PlasmaClientTest plasmaClientTest = new PlasmaClientTest(); + plasmaClientTest.doPlasmaOutOfMemoryExceptionTest(); plasmaClientTest.doByteBufferTest(); plasmaClientTest.doTest(); - } } From 3d0a9d58b6fe29dcb208c3fa244c789449517988 Mon Sep 17 00:00:00 2001 From: Mingyu Zhong Date: Fri, 7 Aug 2020 19:56:01 -0700 Subject: [PATCH 015/460] ARROW-9671: [C++] Fix a bug in BasicDecimal128 constructor that interprets uint64_t integers with highest bit set as negative. Closes #7915 from MingyuZhong/bn Lead-authored-by: Mingyu Zhong Co-authored-by: Micah Kornfield Signed-off-by: Micah Kornfield --- cpp/src/arrow/util/basic_decimal.h | 7 ++++--- cpp/src/arrow/util/decimal_test.cc | 22 ++++++++++++++++------ 2 files changed, 20 insertions(+), 9 deletions(-) diff --git a/cpp/src/arrow/util/basic_decimal.h b/cpp/src/arrow/util/basic_decimal.h index 01feeacdfa14..23c38bbb9d34 100644 --- a/cpp/src/arrow/util/basic_decimal.h +++ b/cpp/src/arrow/util/basic_decimal.h @@ -51,10 +51,11 @@ class ARROW_EXPORT BasicDecimal128 { /// \brief Convert any integer value into a BasicDecimal128. template ::value, T>::type> + typename = typename std::enable_if< + std::is_integral::value && (sizeof(T) <= sizeof(uint64_t)), T>::type> constexpr BasicDecimal128(T value) noexcept - : BasicDecimal128(static_cast(value) >= 0 ? 0 : -1, - static_cast(value)) {} + : BasicDecimal128(value >= T{0} ? 0 : -1, static_cast(value)) { // NOLINT + } /// \brief Create a BasicDecimal128 from an array of bytes. Bytes are assumed to be in /// native-endian byte order. diff --git a/cpp/src/arrow/util/decimal_test.cc b/cpp/src/arrow/util/decimal_test.cc index b62992cff82f..856f10e626b4 100644 --- a/cpp/src/arrow/util/decimal_test.cc +++ b/cpp/src/arrow/util/decimal_test.cc @@ -218,8 +218,7 @@ TEST(DecimalZerosTest, NoLeadingZerosDecimalPoint) { template class Decimal128Test : public ::testing::Test { public: - Decimal128Test() : value_(42) {} - const T value_; + Decimal128Test() {} }; using Decimal128Types = @@ -231,18 +230,29 @@ using Decimal128Types = TYPED_TEST_SUITE(Decimal128Test, Decimal128Types); TYPED_TEST(Decimal128Test, ConstructibleFromAnyIntegerType) { - Decimal128 value(this->value_); - ASSERT_EQ(42, value.low_bits()); + Decimal128 value(TypeParam{42}); + EXPECT_EQ(42, value.low_bits()); + EXPECT_EQ(0, value.high_bits()); + + Decimal128 max_value(std::numeric_limits::max()); + EXPECT_EQ(std::numeric_limits::max(), max_value.low_bits()); + EXPECT_EQ(0, max_value.high_bits()); + + Decimal128 min_value(std::numeric_limits::min()); + EXPECT_EQ(std::numeric_limits::min(), min_value.low_bits()); + EXPECT_EQ((std::is_signed::value ? -1 : 0), min_value.high_bits()); } TEST(Decimal128TestTrue, ConstructibleFromBool) { Decimal128 value(true); - ASSERT_EQ(1, value.low_bits()); + EXPECT_EQ(1, value.low_bits()); + EXPECT_EQ(0, value.high_bits()); } TEST(Decimal128TestFalse, ConstructibleFromBool) { Decimal128 value(false); - ASSERT_EQ(0, value.low_bits()); + EXPECT_EQ(0, value.low_bits()); + EXPECT_EQ(0, value.high_bits()); } TEST(Decimal128Test, Division) { From 40ac6e35346bfc46481e70265390c04f842c2626 Mon Sep 17 00:00:00 2001 From: fredgan Date: Sun, 9 Aug 2020 06:07:13 +0900 Subject: [PATCH 016/460] ARROW-9205: [Documentation] Fix typos Minor issue. Reusing the old bugid. Closes #7824 from fredgan/typos_9205 Authored-by: fredgan Signed-off-by: Sutou Kouhei --- docs/source/format/Columnar.rst | 4 ++-- docs/source/format/Flight.rst | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/source/format/Columnar.rst b/docs/source/format/Columnar.rst index 2232afa22663..f51c6aaf6335 100644 --- a/docs/source/format/Columnar.rst +++ b/docs/source/format/Columnar.rst @@ -807,7 +807,7 @@ of these types: * RecordBatch * DictionaryBatch -We specify a so-called *encapsulated IPC message** format which +We specify a so-called *encapsulated IPC message* format which includes a serialized Flatbuffer type along with an optional message body. We define this message format before describing how to serialize each constituent IPC message type. @@ -1028,7 +1028,7 @@ Schematically we have: :: In the file format, there is no requirement that dictionary keys should be defined in a ``DictionaryBatch`` before they are used in a ``RecordBatch``, as long as the keys are defined somewhere in the -file. Further more, it is invalid to have more then one **non-delta** +file. Further more, it is invalid to have more than one **non-delta** dictionary batch per dictionary ID (i.e. dictionary replacement is not supported). Delta dictionaries are applied in the order they appear in the file footer. diff --git a/docs/source/format/Flight.rst b/docs/source/format/Flight.rst index 257bf6669d4f..c79c563864a0 100644 --- a/docs/source/format/Flight.rst +++ b/docs/source/format/Flight.rst @@ -20,7 +20,7 @@ Arrow Flight RPC ================ -Arrow Flight is a RPC framework for high-performance data services +Arrow Flight is an RPC framework for high-performance data services based on Arrow data, and is built on top of gRPC_ and the :doc:`IPC format `. From 52d0fe618d06df8018c6af96e09e0c4ee538e724 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= Date: Mon, 10 Aug 2020 14:57:54 +0200 Subject: [PATCH 017/460] ARROW-9429: [Python] ChunkedArray.to_numpy MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit While it technically is still using pandas during the conversion it exposes the to_numpy() method. Also refactored the chunked array construction to support more flexible input, see the test case. Closes #7868 from kszucs/ARROW-9429 Lead-authored-by: Krisztián Szűcs Co-authored-by: Antoine Pitrou Signed-off-by: Antoine Pitrou --- python/pyarrow/table.pxi | 68 ++++++++++++------- python/pyarrow/tests/test_extension_type.py | 21 ++++-- python/pyarrow/tests/test_table.py | 74 ++++++++++++++++++++- 3 files changed, 132 insertions(+), 31 deletions(-) diff --git a/python/pyarrow/table.pxi b/python/pyarrow/table.pxi index 37064a51dc55..b8205a31c02d 100644 --- a/python/pyarrow/table.pxi +++ b/python/pyarrow/table.pxi @@ -226,24 +226,35 @@ cdef class ChunkedArray(_PandasConvertible): def _to_pandas(self, options, **kwargs): return _array_like_to_pandas(self, options) - def __array__(self, dtype=None): + def to_numpy(self): + """ + Return a NumPy copy of this array (experimental). + + Returns + ------- + array : numpy.ndarray + """ cdef: PyObject* out PandasOptions c_options object values if self.type.id == _Type_EXTENSION: - return ( - chunked_array( - [self.chunk(i).storage for i in range(self.num_chunks)] - ).__array__(dtype) + storage_array = chunked_array( + [chunk.storage for chunk in self.iterchunks()], + type=self.type.storage_type ) + return storage_array.to_numpy() with nogil: - check_status(libarrow.ConvertChunkedArrayToPandas( - c_options, - self.sp_chunked_array, - self, &out)) + check_status( + ConvertChunkedArrayToPandas( + c_options, + self.sp_chunked_array, + self, + &out + ) + ) # wrap_array_output uses pandas to convert to Categorical, here # always convert to numpy array @@ -252,6 +263,10 @@ cdef class ChunkedArray(_PandasConvertible): if isinstance(values, dict): values = np.take(values['dictionary'], values['indices']) + return values + + def __array__(self, dtype=None): + values = self.to_numpy() if dtype is None: return values return values.astype(dtype) @@ -416,7 +431,6 @@ def chunked_array(arrays, type=None): Must all be the same data type. Can be empty only if type also passed. type : DataType or string coercible to DataType - Returns ------- ChunkedArray @@ -425,31 +439,35 @@ def chunked_array(arrays, type=None): Array arr vector[shared_ptr[CArray]] c_arrays shared_ptr[CChunkedArray] sp_chunked_array - shared_ptr[CDataType] sp_data_type + + type = ensure_type(type, allow_none=True) if isinstance(arrays, Array): arrays = [arrays] for x in arrays: - if isinstance(x, Array): - arr = x - if type is not None: - assert x.type == type + arr = x if isinstance(x, Array) else array(x, type=type) + + if type is None: + # it allows more flexible chunked array construction from to coerce + # subsequent arrays to the firstly inferred array type + # it also spares the inference overhead after the first chunk + type = arr.type else: - arr = array(x, type=type) + if arr.type != type: + raise TypeError( + "All array chunks must have type {}".format(type) + ) c_arrays.push_back(arr.sp_array) - if type: - type = ensure_type(type) - sp_data_type = pyarrow_unwrap_data_type(type) - sp_chunked_array.reset(new CChunkedArray(c_arrays, sp_data_type)) - else: - if c_arrays.size() == 0: - raise ValueError("When passing an empty collection of arrays " - "you must also pass the data type") - sp_chunked_array.reset(new CChunkedArray(c_arrays)) + if c_arrays.size() == 0 and type is None: + raise ValueError("When passing an empty collection of arrays " + "you must also pass the data type") + sp_chunked_array.reset( + new CChunkedArray(c_arrays, pyarrow_unwrap_data_type(type)) + ) with nogil: check_status(sp_chunked_array.get().Validate()) diff --git a/python/pyarrow/tests/test_extension_type.py b/python/pyarrow/tests/test_extension_type.py index dafa4f0d8f39..a3ef336a22f7 100644 --- a/python/pyarrow/tests/test_extension_type.py +++ b/python/pyarrow/tests/test_extension_type.py @@ -482,7 +482,20 @@ def test_to_numpy(): np.testing.assert_array_equal(result, expected) # chunked array - charr = pa.chunked_array([arr]) - - result = np.asarray(charr) - np.testing.assert_array_equal(result, expected) + a1 = pa.chunked_array([arr, arr]) + a2 = pa.chunked_array([arr, arr], type=period_type) + expected = np.hstack([expected, expected]) + + for charr in [a1, a2]: + assert charr.type == period_type + for result in [np.asarray(charr), charr.to_numpy()]: + assert result.dtype == np.int64 + np.testing.assert_array_equal(result, expected) + + # zero chunks + charr = pa.chunked_array([], type=period_type) + assert charr.type == period_type + + for result in [np.asarray(charr), charr.to_numpy()]: + assert result.dtype == np.int64 + np.testing.assert_array_equal(result, np.array([], dtype='int64')) diff --git a/python/pyarrow/tests/test_table.py b/python/pyarrow/tests/test_table.py index af32e180ec1b..4012a9195481 100644 --- a/python/pyarrow/tests/test_table.py +++ b/python/pyarrow/tests/test_table.py @@ -57,9 +57,79 @@ def test_chunked_array_basics(): assert wr() is None +def test_chunked_array_construction(): + arr = pa.chunked_array([ + [1, 2, 3], + [4, 5, 6], + [7, 8, 9], + ]) + assert arr.type == pa.int64() + assert len(arr) == 9 + assert len(arr.chunks) == 3 + + arr = pa.chunked_array([ + [1, 2, 3], + [4., 5., 6.], + [7, 8, 9], + ]) + assert arr.type == pa.int64() + assert len(arr) == 9 + assert len(arr.chunks) == 3 + + arr = pa.chunked_array([ + [1, 2, 3], + [4., 5., 6.], + [7, 8, 9], + ], type=pa.int8()) + assert arr.type == pa.int8() + assert len(arr) == 9 + assert len(arr.chunks) == 3 + + arr = pa.chunked_array([ + [1, 2, 3], + [] + ]) + assert arr.type == pa.int64() + assert len(arr) == 3 + assert len(arr.chunks) == 2 + + msg = ( + "When passing an empty collection of arrays you must also pass the " + "data type" + ) + with pytest.raises(ValueError, match=msg): + assert pa.chunked_array([]) + + assert pa.chunked_array([], type=pa.string()).type == pa.string() + assert pa.chunked_array([[]]).type == pa.null() + assert pa.chunked_array([[]], type=pa.string()).type == pa.string() + + +def test_chunked_array_to_numpy(): + data = pa.chunked_array([ + [1, 2, 3], + [4, 5, 6], + [] + ]) + arr1 = np.asarray(data) + arr2 = data.to_numpy() + + assert isinstance(arr2, np.ndarray) + assert arr2.shape == (6,) + assert np.array_equal(arr1, arr2) + + def test_chunked_array_mismatch_types(): - with pytest.raises(pa.ArrowInvalid): - pa.chunked_array([pa.array([1, 2]), pa.array(['foo', 'bar'])]) + with pytest.raises(TypeError): + # Given array types are different + pa.chunked_array([ + pa.array([1, 2, 3]), + pa.array([1., 2., 3.]) + ]) + + with pytest.raises(TypeError): + # Given array type is different from explicit type argument + pa.chunked_array([pa.array([1, 2, 3])], type=pa.float64()) def test_chunked_array_str(): From 811d8f6c5ca5e5bbf70a4d8c93d4c7fe0a9b9847 Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Mon, 10 Aug 2020 15:33:28 +0200 Subject: [PATCH 018/460] ARROW-9402: [C++] Rework portable wrappers for checked integer arithmetic Vendor relevant code from the portable-snippets library (~ public domain): https://github.com/nemequ/portable-snippets/tree/master/safe-math Also fix some bugs in checked arithmetic (null values had their value slots checked). Add compute scaffolding for stateful binary scalar functions. Closes #7784 from pitrou/ARROW-9402-overflow-arith Authored-by: Antoine Pitrou Signed-off-by: Antoine Pitrou --- LICENSE.txt | 16 + cpp/src/arrow/array/concatenate.cc | 2 +- cpp/src/arrow/array/data.cc | 2 +- cpp/src/arrow/array/validate.cc | 16 +- cpp/src/arrow/buffer.cc | 2 +- .../arrow/compute/kernels/codegen_internal.h | 353 ++++-- .../compute/kernels/scalar_arithmetic.cc | 148 +-- .../kernels/scalar_arithmetic_benchmark.cc | 57 +- .../compute/kernels/scalar_arithmetic_test.cc | 280 +++-- .../compute/kernels/scalar_cast_boolean.cc | 8 +- .../compute/kernels/scalar_cast_numeric.cc | 50 +- .../compute/kernels/scalar_cast_temporal.cc | 6 +- .../arrow/compute/kernels/scalar_nested.cc | 4 +- .../arrow/compute/kernels/scalar_string.cc | 6 +- cpp/src/arrow/compute/kernels/test_util.cc | 113 +- cpp/src/arrow/compute/kernels/test_util.h | 10 + cpp/src/arrow/dataset/filter.cc | 5 +- cpp/src/arrow/pretty_print.cc | 2 +- cpp/src/arrow/scalar.h | 16 +- cpp/src/arrow/util/basic_decimal.cc | 2 +- cpp/src/arrow/util/bit_block_counter.cc | 23 +- cpp/src/arrow/util/bit_block_counter.h | 186 ++- cpp/src/arrow/util/bit_block_counter_test.cc | 92 ++ cpp/src/arrow/util/decimal.cc | 2 +- cpp/src/arrow/util/int_util.h | 94 -- cpp/src/arrow/util/int_util_internal.h | 124 ++ cpp/src/arrow/util/int_util_test.cc | 1 + cpp/src/arrow/util/windows_fixup.h | 11 + .../vendored/portable-snippets/README.md | 10 + .../vendored/portable-snippets/safe-math.h | 1071 +++++++++++++++++ cpp/src/arrow/visitor_inline.h | 90 +- cpp/src/parquet/arrow/reader_internal.cc | 4 +- cpp/src/parquet/column_reader.cc | 24 +- cpp/src/parquet/types.h | 1 - 34 files changed, 2285 insertions(+), 546 deletions(-) create mode 100644 cpp/src/arrow/util/int_util_internal.h create mode 100644 cpp/src/arrow/vendored/portable-snippets/README.md create mode 100644 cpp/src/arrow/vendored/portable-snippets/safe-math.h diff --git a/LICENSE.txt b/LICENSE.txt index 673b146c807c..e79841d212dc 100644 --- a/LICENSE.txt +++ b/LICENSE.txt @@ -2207,3 +2207,19 @@ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +-------------------------------------------------------------------------------- + +The files in cpp/src/arrow/vendored/portable-snippets/ contain code from + +https://github.com/nemequ/portable-snippets + +and have the following copyright notice: + +Each source file contains a preamble explaining the license situation +for that file, which takes priority over this file. With the +exception of some code pulled in from other repositories (such as +µnit, an MIT-licensed project which is used for testing), the code is +public domain, released using the CC0 1.0 Universal dedication (*). + +(*) https://creativecommons.org/publicdomain/zero/1.0/legalcode diff --git a/cpp/src/arrow/array/concatenate.cc b/cpp/src/arrow/array/concatenate.cc index 923bf308bc50..845615453f23 100644 --- a/cpp/src/arrow/array/concatenate.cc +++ b/cpp/src/arrow/array/concatenate.cc @@ -36,7 +36,7 @@ #include "arrow/util/bit_util.h" #include "arrow/util/bitmap_ops.h" #include "arrow/util/checked_cast.h" -#include "arrow/util/int_util.h" +#include "arrow/util/int_util_internal.h" #include "arrow/util/logging.h" #include "arrow/visitor_inline.h" diff --git a/cpp/src/arrow/array/data.cc b/cpp/src/arrow/array/data.cc index 6af1c443ad93..7bfb39532dd4 100644 --- a/cpp/src/arrow/array/data.cc +++ b/cpp/src/arrow/array/data.cc @@ -29,7 +29,7 @@ #include "arrow/status.h" #include "arrow/type.h" #include "arrow/util/bitmap_ops.h" -#include "arrow/util/int_util.h" +#include "arrow/util/int_util_internal.h" #include "arrow/util/logging.h" #include "arrow/util/macros.h" diff --git a/cpp/src/arrow/array/validate.cc b/cpp/src/arrow/array/validate.cc index 8fb8b59b9bcc..4ba6a347d72b 100644 --- a/cpp/src/arrow/array/validate.cc +++ b/cpp/src/arrow/array/validate.cc @@ -26,7 +26,7 @@ #include "arrow/type_traits.h" #include "arrow/util/bit_util.h" #include "arrow/util/checked_cast.h" -#include "arrow/util/int_util.h" +#include "arrow/util/int_util_internal.h" #include "arrow/util/logging.h" #include "arrow/visitor_inline.h" @@ -98,8 +98,9 @@ struct ValidateArrayVisitor { if (value_size < 0) { return Status::Invalid("FixedSizeListArray has negative value size ", value_size); } - if (HasPositiveMultiplyOverflow(len, value_size) || - array.values()->length() != len * value_size) { + int64_t expected_values_length = -1; + if (MultiplyWithOverflow(len, value_size, &expected_values_length) || + array.values()->length() != expected_values_length) { return Status::Invalid("Values Length (", array.values()->length(), ") is not equal to the length (", len, ") multiplied by the value size (", value_size, ")"); @@ -329,7 +330,8 @@ Status ValidateArray(const Array& array) { type.ToString(), ", got ", data.buffers.size()); } // This check is required to avoid addition overflow below - if (HasPositiveAdditionOverflow(array.length(), array.offset())) { + int64_t length_plus_offset = -1; + if (AddWithOverflow(array.length(), array.offset(), &length_plus_offset)) { return Status::Invalid("Array of type ", type.ToString(), " has impossibly large length and offset"); } @@ -343,15 +345,13 @@ Status ValidateArray(const Array& array) { int64_t min_buffer_size = -1; switch (spec.kind) { case DataTypeLayout::BITMAP: - min_buffer_size = BitUtil::BytesForBits(array.length() + array.offset()); + min_buffer_size = BitUtil::BytesForBits(length_plus_offset); break; case DataTypeLayout::FIXED_WIDTH: - if (HasPositiveMultiplyOverflow(array.length() + array.offset(), - spec.byte_width)) { + if (MultiplyWithOverflow(length_plus_offset, spec.byte_width, &min_buffer_size)) { return Status::Invalid("Array of type ", type.ToString(), " has impossibly large length and offset"); } - min_buffer_size = spec.byte_width * (array.length() + array.offset()); break; case DataTypeLayout::ALWAYS_NULL: // XXX Should we raise on non-null buffer? diff --git a/cpp/src/arrow/buffer.cc b/cpp/src/arrow/buffer.cc index 2614cd17f8a3..6aac70eebfd2 100644 --- a/cpp/src/arrow/buffer.cc +++ b/cpp/src/arrow/buffer.cc @@ -25,7 +25,7 @@ #include "arrow/result.h" #include "arrow/status.h" #include "arrow/util/bit_util.h" -#include "arrow/util/int_util.h" +#include "arrow/util/int_util_internal.h" #include "arrow/util/logging.h" #include "arrow/util/string.h" diff --git a/cpp/src/arrow/compute/kernels/codegen_internal.h b/cpp/src/arrow/compute/kernels/codegen_internal.h index 4d8918ae773b..a4e11fe38945 100644 --- a/cpp/src/arrow/compute/kernels/codegen_internal.h +++ b/cpp/src/arrow/compute/kernels/codegen_internal.h @@ -35,6 +35,7 @@ #include "arrow/status.h" #include "arrow/type.h" #include "arrow/type_traits.h" +#include "arrow/util/bit_block_counter.h" #include "arrow/util/bit_util.h" #include "arrow/util/bitmap_generate.h" #include "arrow/util/bitmap_reader.h" @@ -50,10 +51,14 @@ namespace arrow { +using internal::BinaryBitBlockCounter; +using internal::BitBlockCount; using internal::BitmapReader; using internal::checked_cast; using internal::FirstTimeBitmapWriter; using internal::GenerateBitsUnrolled; +using internal::VisitBitBlocksVoid; +using internal::VisitTwoBitBlocksVoid; namespace compute { namespace internal { @@ -121,6 +126,57 @@ struct OptionsWrapper : public KernelState { OptionsType options; }; +// ---------------------------------------------------------------------- +// Input and output value type definitions + +template +struct GetViewType; + +template +struct GetViewType> { + using T = typename Type::c_type; + using PhysicalType = T; + + static T LogicalValue(PhysicalType value) { return value; } +}; + +template +struct GetViewType::value || + is_fixed_size_binary_type::value>> { + using T = util::string_view; + using PhysicalType = T; + + static T LogicalValue(PhysicalType value) { return value; } +}; + +template <> +struct GetViewType { + using T = Decimal128; + using PhysicalType = util::string_view; + + static T LogicalValue(PhysicalType value) { + return Decimal128(reinterpret_cast(value.data())); + } +}; + +template +struct GetOutputType; + +template +struct GetOutputType> { + using T = typename Type::c_type; +}; + +template +struct GetOutputType::value>> { + using T = std::string; +}; + +template <> +struct GetOutputType { + using T = Decimal128; +}; + // ---------------------------------------------------------------------- // Iteration / value access utilities @@ -128,6 +184,8 @@ template using enable_if_has_c_type_not_boolean = enable_if_t::value && !is_boolean_type::value, R>; +// Iterator over various input array types, yielding a GetViewType + template struct ArrayIterator; @@ -135,6 +193,7 @@ template struct ArrayIterator> { using T = typename Type::c_type; const T* values; + explicit ArrayIterator(const ArrayData& data) : values(data.GetValues(1)) {} T operator()() { return *values++; } }; @@ -142,6 +201,7 @@ struct ArrayIterator> { template struct ArrayIterator> { BitmapReader reader; + explicit ArrayIterator(const ArrayData& data) : reader(data.buffers[1]->data(), data.offset, data.length) {} bool operator()() { @@ -159,6 +219,7 @@ struct ArrayIterator> { offset_type cur_offset; const char* data; int64_t position; + explicit ArrayIterator(const ArrayData& arr) : arr(arr), offsets(reinterpret_cast(arr.buffers[1]->data()) + @@ -168,13 +229,34 @@ struct ArrayIterator> { position(0) {} util::string_view operator()() { - offset_type next_offset = offsets[position++ + 1]; + offset_type next_offset = offsets[++position]; auto result = util::string_view(data + cur_offset, next_offset - cur_offset); cur_offset = next_offset; return result; } }; +// Iterator over various output array types, taking a GetOutputType + +template +struct OutputArrayWriter; + +template +struct OutputArrayWriter> { + using T = typename Type::c_type; + T* values; + + explicit OutputArrayWriter(ArrayData* data) : values(data->GetMutableValues(1)) {} + + void Write(T value) { *values++ = value; } + + // Note that this doesn't write the null bitmap, which should be consistent + // with Write / WriteNull calls + void WriteNull() { *values++ = T{}; } +}; + +// (Un)box Scalar to / from C++ value + template struct UnboxScalar; @@ -201,43 +283,6 @@ struct UnboxScalar { } }; -template -struct GetViewType; - -template -struct GetViewType> { - using T = typename Type::c_type; -}; - -template -struct GetViewType::value || - is_fixed_size_binary_type::value>> { - using T = util::string_view; -}; - -template <> -struct GetViewType { - using T = Decimal128; -}; - -template -struct GetOutputType; - -template -struct GetOutputType> { - using T = typename Type::c_type; -}; - -template -struct GetOutputType::value>> { - using T = std::string; -}; - -template <> -struct GetOutputType { - using T = Decimal128; -}; - template struct BoxScalar; @@ -264,6 +309,41 @@ struct BoxScalar { static void Box(T val, Scalar* out) { checked_cast(out)->value = val; } }; +// A VisitArrayDataInline variant that calls its visitor function with logical +// values, such as Decimal128 rather than util::string_view. + +template +static void VisitArrayValuesInline(const ArrayData& arr, VisitFunc&& valid_func, + NullFunc&& null_func) { + VisitArrayDataInline( + arr, + [&](typename GetViewType::PhysicalType v) { + valid_func(GetViewType::LogicalValue(std::move(v))); + }, + std::forward(null_func)); +} + +// Like VisitArrayValuesInline, but for binary functions. + +template +static void VisitTwoArrayValuesInline(const ArrayData& arr0, const ArrayData& arr1, + VisitFunc&& valid_func, NullFunc&& null_func) { + ArrayIterator arr0_it(arr0); + ArrayIterator arr1_it(arr1); + + auto visit_valid = [&](int64_t i) { + valid_func(GetViewType::LogicalValue(arr0_it()), + GetViewType::LogicalValue(arr1_it())); + }; + auto visit_null = [&]() { + arr0_it(); + arr1_it(); + null_func(); + }; + VisitTwoBitBlocksVoid(arr0.buffers[0], arr0.offset, arr1.buffers[0], arr1.offset, + arr0.length, std::move(visit_valid), std::move(visit_null)); +} + // ---------------------------------------------------------------------- // Reusable type resolvers @@ -406,26 +486,27 @@ struct OutputAdapter> { // The "Op" functor should have the form // // struct Op { -// template -// static OUT Call(KernelContext* ctx, ARG0 val) { +// template +// static OutValue Call(KernelContext* ctx, Arg0Value val) { // // implementation // } // }; template struct ScalarUnary { - using OUT = typename GetOutputType::T; - using ARG0 = typename GetViewType::T; + using OutValue = typename GetOutputType::T; + using Arg0Value = typename GetViewType::T; static void Array(KernelContext* ctx, const ArrayData& arg0, Datum* out) { ArrayIterator arg0_it(arg0); - OutputAdapter::Write( - ctx, out, [&]() -> OUT { return Op::template Call(ctx, arg0_it()); }); + OutputAdapter::Write(ctx, out, [&]() -> OutValue { + return Op::template Call(ctx, arg0_it()); + }); } static void Scalar(KernelContext* ctx, const Scalar& arg0, Datum* out) { if (arg0.is_valid) { - ARG0 arg0_val = UnboxScalar::Unbox(arg0); - BoxScalar::Box(Op::template Call(ctx, arg0_val), + Arg0Value arg0_val = UnboxScalar::Unbox(arg0); + BoxScalar::Box(Op::template Call(ctx, arg0_val), out->scalar().get()); } else { out->value = MakeNullScalar(arg0.type); @@ -441,37 +522,13 @@ struct ScalarUnary { } }; -// A VisitArrayDataInline variant that passes a Decimal128 value, -// not util::string_view, for decimal128 arrays, - -template -static typename std::enable_if::value, void>::type -VisitArrayValuesInline(const ArrayData& arr, VisitFunc&& valid_func, - NullFunc&& null_func) { - VisitArrayDataInline(arr, std::forward(valid_func), - std::forward(null_func)); -} - -template -static typename std::enable_if::value, void>::type -VisitArrayValuesInline(const ArrayData& arr, VisitFunc&& valid_func, - NullFunc&& null_func) { - VisitArrayDataInline( - arr, - [&](util::string_view v) { - const auto dec_value = Decimal128(reinterpret_cast(v.data())); - valid_func(dec_value); - }, - std::forward(null_func)); -} - // An alternative to ScalarUnary that Applies a scalar operation with state on // only the not-null values of a single array template struct ScalarUnaryNotNullStateful { using ThisType = ScalarUnaryNotNullStateful; - using OUT = typename GetOutputType::T; - using ARG0 = typename GetViewType::T; + using OutValue = typename GetOutputType::T; + using Arg0Value = typename GetViewType::T; Op op; explicit ScalarUnaryNotNullStateful(Op op) : op(std::move(op)) {} @@ -493,10 +550,12 @@ struct ScalarUnaryNotNullStateful { static void Exec(const ThisType& functor, KernelContext* ctx, const ArrayData& arg0, Datum* out) { ArrayData* out_arr = out->mutable_array(); - auto out_data = out_arr->GetMutableValues(1); + auto out_data = out_arr->GetMutableValues(1); VisitArrayValuesInline( arg0, - [&](ARG0 v) { *out_data++ = functor.op.template Call(ctx, v); }, + [&](Arg0Value v) { + *out_data++ = functor.op.template Call(ctx, v); + }, [&]() { // null ++out_data; @@ -515,7 +574,7 @@ struct ScalarUnaryNotNullStateful { typename TypeTraits::BuilderType builder; VisitArrayValuesInline( arg0, - [&](ARG0 v) { + [&](Arg0Value v) { KERNEL_RETURN_IF_ERROR(ctx, builder.Append(functor.op.Call(ctx, v))); }, [&]() { KERNEL_RETURN_IF_ERROR(ctx, builder.AppendNull()); }); @@ -536,8 +595,8 @@ struct ScalarUnaryNotNullStateful { out_arr->offset, out_arr->length); VisitArrayValuesInline( arg0, - [&](ARG0 v) { - if (functor.op.template Call(ctx, v)) { + [&](Arg0Value v) { + if (functor.op.template Call(ctx, v)) { out_writer.Set(); } out_writer.Next(); @@ -559,8 +618,8 @@ struct ScalarUnaryNotNullStateful { auto out_data = out_arr->GetMutableValues(1); VisitArrayValuesInline( arg0, - [&](ARG0 v) { - functor.op.template Call(ctx, v).ToBytes(out_data); + [&](Arg0Value v) { + functor.op.template Call(ctx, v).ToBytes(out_data); out_data += 16; }, [&]() { out_data += 16; }); @@ -569,8 +628,8 @@ struct ScalarUnaryNotNullStateful { void Scalar(KernelContext* ctx, const Scalar& arg0, Datum* out) { if (arg0.is_valid) { - ARG0 arg0_val = UnboxScalar::Unbox(arg0); - BoxScalar::Box(this->op.template Call(ctx, arg0_val), + Arg0Value arg0_val = UnboxScalar::Unbox(arg0); + BoxScalar::Box(this->op.template Call(ctx, arg0_val), out->scalar().get()); } else { out->value = MakeNullScalar(arg0.type); @@ -591,8 +650,8 @@ struct ScalarUnaryNotNullStateful { // operator requires some initialization use ScalarUnaryNotNullStateful template struct ScalarUnaryNotNull { - using OUT = typename GetOutputType::T; - using ARG0 = typename GetViewType::T; + using OutValue = typename GetOutputType::T; + using Arg0Value = typename GetViewType::T; static void Exec(KernelContext* ctx, const ExecBatch& batch, Datum* out) { // Seed kernel with dummy state @@ -612,39 +671,42 @@ struct ScalarUnaryNotNull { // The "Op" functor should have the form // // struct Op { -// template -// static OUT Call(KernelContext* ctx, ARG0 arg0, ARG1 arg1) { +// template +// static OutValue Call(KernelContext* ctx, Arg0Value arg0, Arg1Value arg1) { // // implementation // } // }; template struct ScalarBinary { - using OUT = typename GetOutputType::T; - using ARG0 = typename GetViewType::T; - using ARG1 = typename GetViewType::T; + using OutValue = typename GetOutputType::T; + using Arg0Value = typename GetViewType::T; + using Arg1Value = typename GetViewType::T; static void ArrayArray(KernelContext* ctx, const ArrayData& arg0, const ArrayData& arg1, Datum* out) { ArrayIterator arg0_it(arg0); ArrayIterator arg1_it(arg1); - OutputAdapter::Write( - ctx, out, [&]() -> OUT { return Op::template Call(ctx, arg0_it(), arg1_it()); }); + OutputAdapter::Write(ctx, out, [&]() -> OutValue { + return Op::template Call(ctx, arg0_it(), arg1_it()); + }); } static void ArrayScalar(KernelContext* ctx, const ArrayData& arg0, const Scalar& arg1, Datum* out) { ArrayIterator arg0_it(arg0); auto arg1_val = UnboxScalar::Unbox(arg1); - OutputAdapter::Write( - ctx, out, [&]() -> OUT { return Op::template Call(ctx, arg0_it(), arg1_val); }); + OutputAdapter::Write(ctx, out, [&]() -> OutValue { + return Op::template Call(ctx, arg0_it(), arg1_val); + }); } static void ScalarArray(KernelContext* ctx, const Scalar& arg0, const ArrayData& arg1, Datum* out) { auto arg0_val = UnboxScalar::Unbox(arg0); ArrayIterator arg1_it(arg1); - OutputAdapter::Write( - ctx, out, [&]() -> OUT { return Op::template Call(ctx, arg0_val, arg1_it()); }); + OutputAdapter::Write(ctx, out, [&]() -> OutValue { + return Op::template Call(ctx, arg0_val, arg1_it()); + }); } static void ScalarScalar(KernelContext* ctx, const Scalar& arg0, const Scalar& arg1, @@ -675,11 +737,114 @@ struct ScalarBinary { } }; +// An alternative to ScalarBinary that Applies a scalar operation with state on +// only the value pairs which are not-null in both arrays +template +struct ScalarBinaryNotNullStateful { + using ThisType = ScalarBinaryNotNullStateful; + using OutValue = typename GetOutputType::T; + using Arg0Value = typename GetViewType::T; + using Arg1Value = typename GetViewType::T; + + Op op; + explicit ScalarBinaryNotNullStateful(Op op) : op(std::move(op)) {} + + // NOTE: In ArrayExec, Type is really OutputType + + void ArrayArray(KernelContext* ctx, const ArrayData& arg0, const ArrayData& arg1, + Datum* out) { + OutputArrayWriter writer(out->mutable_array()); + VisitTwoArrayValuesInline( + arg0, arg1, + [&](Arg0Value u, Arg1Value v) { + writer.Write(op.template Call(ctx, u, v)); + }, + [&]() { writer.WriteNull(); }); + } + + void ArrayScalar(KernelContext* ctx, const ArrayData& arg0, const Scalar& arg1, + Datum* out) { + OutputArrayWriter writer(out->mutable_array()); + if (arg1.is_valid) { + const auto arg1_val = UnboxScalar::Unbox(arg1); + VisitArrayValuesInline( + arg0, + [&](Arg0Value u) { + writer.Write( + op.template Call(ctx, u, arg1_val)); + }, + [&]() { writer.WriteNull(); }); + } + } + + void ScalarArray(KernelContext* ctx, const Scalar& arg0, const ArrayData& arg1, + Datum* out) { + OutputArrayWriter writer(out->mutable_array()); + if (arg0.is_valid) { + const auto arg0_val = UnboxScalar::Unbox(arg0); + VisitArrayValuesInline( + arg1, + [&](Arg1Value v) { + writer.Write( + op.template Call(ctx, arg0_val, v)); + }, + [&]() { writer.WriteNull(); }); + } + } + + void ScalarScalar(KernelContext* ctx, const Scalar& arg0, const Scalar& arg1, + Datum* out) { + if (arg0.is_valid && arg1.is_valid) { + const auto arg0_val = UnboxScalar::Unbox(arg0); + const auto arg1_val = UnboxScalar::Unbox(arg1); + BoxScalar::Box( + op.template Call(ctx, arg0_val, arg1_val), + out->scalar().get()); + } + } + + void Exec(KernelContext* ctx, const ExecBatch& batch, Datum* out) { + if (batch[0].kind() == Datum::ARRAY) { + if (batch[1].kind() == Datum::ARRAY) { + return ArrayArray(ctx, *batch[0].array(), *batch[1].array(), out); + } else { + return ArrayScalar(ctx, *batch[0].array(), *batch[1].scalar(), out); + } + } else { + if (batch[1].kind() == Datum::ARRAY) { + return ScalarArray(ctx, *batch[0].scalar(), *batch[1].array(), out); + } else { + return ScalarScalar(ctx, *batch[0].scalar(), *batch[1].scalar(), out); + } + } + } +}; + +// An alternative to ScalarBinary that Applies a scalar operation on only +// the value pairs which are not-null in both arrays. +// The operator is not stateful; if the operator requires some initialization +// use ScalarBinaryNotNullStateful. +template +struct ScalarBinaryNotNull { + using OutValue = typename GetOutputType::T; + using Arg0Value = typename GetViewType::T; + using Arg1Value = typename GetViewType::T; + + static void Exec(KernelContext* ctx, const ExecBatch& batch, Datum* out) { + // Seed kernel with dummy state + ScalarBinaryNotNullStateful kernel({}); + return kernel.Exec(ctx, batch, out); + } +}; + // A kernel exec generator for binary kernels where both input types are the // same template using ScalarBinaryEqualTypes = ScalarBinary; +template +using ScalarBinaryNotNullEqualTypes = ScalarBinaryNotNull; + } // namespace applicator // ---------------------------------------------------------------------- diff --git a/cpp/src/arrow/compute/kernels/scalar_arithmetic.cc b/cpp/src/arrow/compute/kernels/scalar_arithmetic.cc index 1f0cd3785a74..299f652dc3d0 100644 --- a/cpp/src/arrow/compute/kernels/scalar_arithmetic.cc +++ b/cpp/src/arrow/compute/kernels/scalar_arithmetic.cc @@ -16,16 +16,21 @@ // under the License. #include "arrow/compute/kernels/common.h" -#include "arrow/util/int_util.h" +#include "arrow/util/int_util_internal.h" #include "arrow/util/macros.h" -#ifndef __has_builtin -#define __has_builtin(x) 0 -#endif - namespace arrow { + +using internal::AddWithOverflow; +using internal::MultiplyWithOverflow; +using internal::SubtractWithOverflow; + namespace compute { namespace internal { + +using applicator::ScalarBinaryEqualTypes; +using applicator::ScalarBinaryNotNullEqualTypes; + namespace { template @@ -72,35 +77,19 @@ struct Add { }; struct AddChecked { -#if __has_builtin(__builtin_add_overflow) - template - static enable_if_integer Call(KernelContext* ctx, T left, T right) { + template + enable_if_integer Call(KernelContext* ctx, Arg0 left, Arg1 right) { + static_assert(std::is_same::value && std::is_same::value, ""); T result; - if (ARROW_PREDICT_FALSE(__builtin_add_overflow(left, right, &result))) { + if (ARROW_PREDICT_FALSE(AddWithOverflow(left, right, &result))) { ctx->SetStatus(Status::Invalid("overflow")); } return result; } -#else - template - static enable_if_unsigned_integer Call(KernelContext* ctx, T left, T right) { - if (ARROW_PREDICT_FALSE(arrow::internal::HasPositiveAdditionOverflow(left, right))) { - ctx->SetStatus(Status::Invalid("overflow")); - } - return left + right; - } - - template - static enable_if_signed_integer Call(KernelContext* ctx, T left, T right) { - if (ARROW_PREDICT_FALSE(arrow::internal::HasSignedAdditionOverflow(left, right))) { - ctx->SetStatus(Status::Invalid("overflow")); - } - return left + right; - } -#endif - template - static constexpr enable_if_floating_point Call(KernelContext*, T left, T right) { + template + enable_if_floating_point Call(KernelContext*, Arg0 left, Arg1 right) { + static_assert(std::is_same::value && std::is_same::value, ""); return left + right; } }; @@ -123,36 +112,19 @@ struct Subtract { }; struct SubtractChecked { -#if __has_builtin(__builtin_sub_overflow) - template - static enable_if_integer Call(KernelContext* ctx, T left, T right) { + template + enable_if_integer Call(KernelContext* ctx, Arg0 left, Arg1 right) { + static_assert(std::is_same::value && std::is_same::value, ""); T result; - if (ARROW_PREDICT_FALSE(__builtin_sub_overflow(left, right, &result))) { + if (ARROW_PREDICT_FALSE(SubtractWithOverflow(left, right, &result))) { ctx->SetStatus(Status::Invalid("overflow")); } return result; } -#else - template - static enable_if_unsigned_integer Call(KernelContext* ctx, T left, T right) { - if (ARROW_PREDICT_FALSE( - arrow::internal::HasPositiveSubtractionOverflow(left, right))) { - ctx->SetStatus(Status::Invalid("overflow")); - } - return left - right; - } - template - static enable_if_signed_integer Call(KernelContext* ctx, T left, T right) { - if (ARROW_PREDICT_FALSE(arrow::internal::HasSignedSubtractionOverflow(left, right))) { - ctx->SetStatus(Status::Invalid("overflow")); - } - return left - right; - } -#endif - - template - static constexpr enable_if_floating_point Call(KernelContext*, T left, T right) { + template + enable_if_floating_point Call(KernelContext*, Arg0 left, Arg1 right) { + static_assert(std::is_same::value && std::is_same::value, ""); return left - right; } }; @@ -197,59 +169,48 @@ struct Multiply { }; struct MultiplyChecked { - template - static enable_if_integer Call(KernelContext* ctx, T left, T right) { + template + enable_if_integer Call(KernelContext* ctx, Arg0 left, Arg1 right) { + static_assert(std::is_same::value && std::is_same::value, ""); T result; -#if __has_builtin(__builtin_mul_overflow) - if (ARROW_PREDICT_FALSE(__builtin_mul_overflow(left, right, &result))) { - ctx->SetStatus(Status::Invalid("overflow")); - } -#else - result = Multiply::Call(ctx, left, right); - if (left != 0 && ARROW_PREDICT_FALSE(result / left != right)) { + if (ARROW_PREDICT_FALSE(MultiplyWithOverflow(left, right, &result))) { ctx->SetStatus(Status::Invalid("overflow")); } -#endif return result; } - template - static constexpr enable_if_floating_point Call(KernelContext*, T left, T right) { + template + enable_if_floating_point Call(KernelContext*, Arg0 left, Arg1 right) { + static_assert(std::is_same::value && std::is_same::value, ""); return left * right; } }; -using applicator::ScalarBinaryEqualTypes; - // Generate a kernel given an arithmetic functor -// -// To avoid undefined behaviour of signed integer overflow treat the signed -// input argument values as unsigned then cast them to signed making them wrap -// around. -template +template