ARROW-11340: [C++] Add vcpkg.json manifest to cpp project root

ci/docker/python-wheel-manylinux-201x.dockerfile

@@ -62,7 +62,8 @@ ARG build_type=release
 ENV CMAKE_BUILD_TYPE=${build_type} \
  VCPKG_FORCE_SYSTEM_BINARIES=1 \
  VCPKG_OVERLAY_TRIPLETS=/arrow/ci/vcpkg \
- VCPKG_DEFAULT_TRIPLET=x64-linux-static-${build_type}
+ VCPKG_DEFAULT_TRIPLET=x64-linux-static-${build_type} \
+ VCPKG_FEATURE_FLAGS=-manifests
 
 # TODO(kszucs): factor out the package enumeration to a text file and reuse it
 # from the windows image and potentially in a future macos wheel build

ci/docker/python-wheel-windows-vs2017.dockerfile

@@ -46,7 +46,8 @@ COPY ci/vcpkg arrow/ci/vcpkg
 ARG build_type=release
 ENV CMAKE_BUILD_TYPE=${build_type} \
  VCPKG_OVERLAY_TRIPLETS=C:\\arrow\\ci\\vcpkg \
- VCPKG_DEFAULT_TRIPLET=x64-windows-static-md-${build_type}
+ VCPKG_DEFAULT_TRIPLET=x64-windows-static-md-${build_type} \
+ VCPKG_FEATURE_FLAGS=-manifests
 RUN vcpkg install --clean-after-build \
  abseil \
  aws-sdk-cpp[config,cognito-identity,core,identity-management,s3,sts,transfer] \

ci/scripts/python_wheel_manylinux_build.sh

@@ -67,6 +67,7 @@ echo "=== (${PYTHON_VERSION}) Building Arrow C++ libraries ==="
 : ${ARROW_WITH_ZSTD:=ON}
 : ${CMAKE_BUILD_TYPE:=release}
 : ${CMAKE_GENERATOR:=Ninja}
+: ${VCPKG_FEATURE_FLAGS:=-manifests}
 
 mkdir /tmp/arrow-build
 pushd /tmp/arrow-build
@@ -106,6 +107,8 @@ cmake \
  -DCMAKE_UNITY_BUILD=ON \
  -DOPENSSL_USE_STATIC_LIBS=ON \
  -DThrift_ROOT=/opt/vcpkg/installed/x64-linux/lib \
+ -D_VCPKG_INSTALLED_DIR=/opt/vcpkg/installed \
+ -DVCPKG_MANIFEST_MODE=OFF \
  -DVCPKG_TARGET_TRIPLET=x64-linux-static-${CMAKE_BUILD_TYPE} \
  -G ${CMAKE_GENERATOR} \
  /arrow/cpp

ci/scripts/python_wheel_windows_build.bat

@@ -47,6 +47,7 @@ set ARROW_WITH_ZLIB=ON
 set ARROW_WITH_ZSTD=ON
 set CMAKE_UNITY_BUILD=ON
 set CMAKE_GENERATOR=Visual Studio 15 2017 Win64
+set VCPKG_FEATURE_FLAGS=-manifests
 
 mkdir C:\arrow-build
 pushd C:\arrow-build
@@ -83,6 +84,8 @@ cmake ^
  -DCMAKE_TOOLCHAIN_FILE=C:\vcpkg\scripts\buildsystems\vcpkg.cmake ^
  -DCMAKE_UNITY_BUILD=%CMAKE_UNITY_BUILD% ^
  -DMSVC_LINK_VERBOSE=ON ^
+ -D_VCPKG_INSTALLED_DIR=C:\vcpkg\installed ^
+ -DVCPKG_MANIFEST_MODE=OFF ^
  -DVCPKG_TARGET_TRIPLET=x64-windows-static-md-%CMAKE_BUILD_TYPE% ^
  -G "%CMAKE_GENERATOR%" ^
  C:\arrow\cpp || exit /B

cpp/CMakeSettings.json

@@ -0,0 +1,21 @@
+{
+ "configurations": [
+ {
+ "name": "x64-Debug (default)",
+ "generator": "Ninja",
+ "configurationType": "Debug",
+ "inheritEnvironments": [ "msvc_x64_x64" ],
+ "buildRoot": "${projectDir}\\out\\build\\${name}",
+ "installRoot": "${projectDir}\\out\\install\\${name}",
+ "cmakeCommandArgs": "",
+ "buildCommandArgs": "",
+ "ctestCommandArgs": "",
+ "variables": [
+ {
+ "name":"VCPKG_MANIFEST_MODE",
+ "value":"OFF"
+ }
+ ]
+ }
+ ]
+}

cpp/vcpkg.json

@@ -0,0 +1,40 @@
+{
+ "name": "arrow",
+ "version-string": "4.0.0-SNAPSHOT",
+ "dependencies": [
+ "abseil",
+ {
+ "name": "aws-sdk-cpp",
+ "features": [
+ "config",
+ "cognito-identity",
+ "identity-management",
+ "s3",
+ "sts",
+ "transfer"
+ ]
+ },
+ "benchmark",
+ "boost",
+ "brotli",
+ "bzip2",
+ "c-ares",
+ "curl",
+ "flatbuffers",
+ "gflags",
+ "glog",
+ "grpc",
+ "gtest",
+ "lz4",
+ "openssl",
+ "orc",
+ "protobuf",
+ "rapidjson",
+ "re2",
+ "snappy",
+ "thrift",
+ "utf8proc",
+ "zlib",
+ "zstd"
+ ]
+}

dev/release/00-prepare-test.rb

@@ -164,6 +164,13 @@ def test_version_pre_tag
  "+set(ARROW_VERSION \"#{@release_version}\")"],
  ],
  },
+ {
+ path: "cpp/vcpkg.json",
+ hunks: [
+ ["- \"version-string\": \"#{@snapshot_version}\",",
+ "+ \"version-string\": \"#{@release_version}\","],
+ ],
+ },
  {
  path: "csharp/Directory.Build.props",
  hunks: [
@@ -410,6 +417,13 @@ def test_version_post_tag
  "+set(ARROW_VERSION \"#{@next_snapshot_version}\")"],
  ],
  },
+ {
+ path: "cpp/vcpkg.json",
+ hunks: [
+ ["- \"version-string\": \"#{@release_version}\",",
+ "+ \"version-string\": \"#{@next_snapshot_version}\","],
+ ],
+ },
  {
  path: "csharp/Directory.Build.props",
  hunks: [

dev/release/00-prepare.sh

@@ -67,6 +67,12 @@ update_versions() {
  CMakeLists.txt
  rm -f CMakeLists.txt.bak
  git add CMakeLists.txt
+
+ sed -i.bak -E -e \
+ "s/\"version-string\": \".+\"/\"version-string\": \"${version}\"/" \
+ vcpkg.json
+ rm -f vcpkg.json.bak
+ git add vcpkg.json
  cd -
 
  cd "${SOURCE_DIR}/../../csharp"

dev/tasks/tasks.yml

@@ -82,6 +82,9 @@ groups:
  ruby:
  - test-*ruby*
 
+ vcpkg:
+ - test-*vcpkg*
+
  integration:
  - test-*dask*
  - test-*hdfs*
@@ -1809,6 +1812,12 @@ tasks:
  UBUNTU: 18.04
  run: ubuntu-docs
 
+ ############################## vcpkg tests ##################################
+
+ test-build-vcpkg-win:
+ ci: github
+ template: vcpkg-tests/github.windows.yml
+
  ############################## Integration tests ############################
 
  test-conda-python-3.7-pandas-latest:

dev/tasks/vcpkg-tests/cpp-build-vcpkg.bat

@@ -0,0 +1,98 @@
+@rem Licensed to the Apache Software Foundation (ASF) under one
+@rem or more contributor license agreements. See the NOTICE file
+@rem distributed with this work for additional information
+@rem regarding copyright ownership. The ASF licenses this file
+@rem to you under the Apache License, Version 2.0 (the
+@rem "License"); you may not use this file except in compliance
+@rem with the License. You may obtain a copy of the License at
+@rem
+@rem http://www.apache.org/licenses/LICENSE-2.0
+@rem
+@rem Unless required by applicable law or agreed to in writing,
+@rem software distributed under the License is distributed on an
+@rem "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+@rem KIND, either express or implied. See the License for the
+@rem specific language governing permissions and limitations
+@rem under the License.
+
+@rem Run VsDevCmd.bat to set Visual Studio environment variables for building
+@rem on the command line. This is the path for Visual Studio Enterprise 2019
+
+call "C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\Common7\Tools\VsDevCmd.bat" -arch=amd64
+
+
+@rem Install build dependencies with vcpkg
+
+@rem TODO(ianmcook): change --x-manifest-root to --manifest-root after it
+@rem changes in vcpkg
+
+vcpkg install ^
+ --triplet x64-windows ^
+ --x-manifest-root cpp ^
+ --clean-after-build ^
+ || exit /B 1
+
+
+@rem Set environment variables
+
+set ARROW_TEST_DATA=%cd%\testing\data
+set PARQUET_TEST_DATA=%cd%\cpp\submodules\parquet-testing\data
+set VCPKG_INSTALLED=%cd%\cpp\vcpkg_installed
+
+
+@rem Build Arrow C++ library
+
+mkdir cpp\build
+pushd cpp\build
+
+@rem TODO(ianmcook): test using --parallel %NUMBER_OF_PROCESSORS% with
+@rem cmake --build instead of specifying -DARROW_CXXFLAGS="/MP" here
+@rem (see https://gitlab.kitware.com/cmake/cmake/-/issues/20564)
+
+@rem TODO(ianmcook): Add -DARROW_BUILD_BENCHMARKS=ON after the issue described
+@rem at https://github.com/google/benchmark/issues/1046 is resolved
+
+cmake -G "Visual Studio 16 2019" -A x64 ^
+ -DARROW_BOOST_USE_SHARED=ON ^
+ -DARROW_BUILD_SHARED=ON ^
+ -DARROW_BUILD_STATIC=OFF ^
+ -DARROW_BUILD_TESTS=ON ^
+ -DARROW_CXXFLAGS="/MP" ^
+ -DARROW_DATASET=ON ^
+ -DARROW_DEPENDENCY_SOURCE=SYSTEM ^
+ -DARROW_FLIGHT=ON ^
+ -DARROW_MIMALLOC=ON ^
+ -DARROW_PACKAGE_PREFIX="%VCPKG_INSTALLED%\x64-windows" ^
+ -DARROW_PARQUET=ON ^
+ -DARROW_PYTHON=OFF ^
+ -DARROW_WITH_BROTLI=ON ^
+ -DARROW_WITH_BZ2=ON ^
+ -DARROW_WITH_LZ4=ON ^
+ -DARROW_WITH_SNAPPY=ON ^
+ -DARROW_WITH_ZLIB=ON ^
+ -DARROW_WITH_ZSTD=ON ^
+ -DCMAKE_BUILD_TYPE=release ^
+ -DCMAKE_TOOLCHAIN_FILE="C:\vcpkg\scripts\buildsystems\vcpkg.cmake" ^
+ -DCMAKE_UNITY_BUILD=ON ^
+ -DLZ4_MSVC_LIB_PREFIX="" ^
+ -DLZ4_MSVC_STATIC_LIB_SUFFIX="" ^
+ -D_VCPKG_INSTALLED_DIR="%VCPKG_INSTALLED%" ^
+ -DVCPKG_MANIFEST_MODE=ON ^
+ -DVCPKG_TARGET_TRIPLET="x64-windows" ^
+ -DZSTD_MSVC_LIB_PREFIX="" ^
+ .. || exit /B 1
+
+cmake --build . --target INSTALL --config Release || exit /B 1
+
+
+@rem Test Arrow C++ library
+
+@rem TODO(ianmcook): Troubleshoot two test failures:
+@rem - TestStatisticsSortOrder/0.MinMax
+@rem - TestStatistic.Int32Extremums
+
+ctest --output-on-failure ^
+ --parallel %NUMBER_OF_PROCESSORS% ^
+ --timeout 300 || exit /B 1
+
+popd

dev/tasks/vcpkg-tests/github.windows.yml

@@ -0,0 +1,63 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# NOTE: must set "Crossbow" as name to have the badge links working in the
+# github comment reports!
+name: Crossbow
+
+on:
+ push:
+ branches:
+ - "*-github-*"
+
+jobs:
+ test-vcpkg-win:
+ name: Install build deps with vcpkg and build Arrow C++
+ runs-on: windows-2019
+ steps:
+ - name: Checkout Arrow
+ run: |
+ git clone --no-checkout {{ arrow.remote }} arrow
+ git -C arrow fetch -t {{ arrow.remote }} {{ arrow.branch }}
+ git -C arrow checkout FETCH_HEAD
+ git -C arrow submodule update --init --recursive
+ - name: Remove and Reinstall vcpkg
+ # As of January 2021, the version of vcpkg that is preinstalled on the
+ # Github Actions windows-2019 image is 2020.11.12, as noted at
+ # https://github.com/actions/virtual-environments/blob/main/images/win/Windows2019-Readme.md
+ # This version of vcpkg has a bug that causes the installation of
+ # aws-cpp-sdk to fail. See details at
+ # https://github.com/awslabs/aws-c-common/issues/734
+ # and https://github.com/microsoft/vcpkg/pull/14716.
+ # When running vcpkg in Github Actions on Windows, remove the
+ # preinstalled vcpkg and install the newest version from source.
+ shell: cmd
+ run: |
+ CALL vcpkg integrate remove 2>NUL
+ CALL C:
+ CALL cd \
+ CALL rmdir /s /q vcpkg 2>NUL
+ CALL git clone https://github.com/microsoft/vcpkg.git vcpkg
+ CALL cd vcpkg
+ CALL bootstrap-vcpkg.bat -win64 -disableMetrics
+ CALL vcpkg integrate install
+ CALL setx PATH "%PATH%;C:\vcpkg"
+ - name: Install Dependencies with vcpkg and Build Arrow C++
+ shell: cmd
+ run: |
+ CALL cd arrow
+ CALL dev\tasks\vcpkg-tests\cpp-build-vcpkg.bat