ARROW-15417: [Python][Packaging] Use vcpkg manifest to install wheel dependencies; downgrade AWS SDK by building the bundled version

ci/docker/java-jni-manylinux-201x.dockerfile

@@ -19,19 +19,16 @@ ARG base
 FROM ${base}
 
 # Install the libaries required by the Gandiva to run
-RUN vcpkg install --clean-after-build \
- boost-algorithm \
- boost-crc \
- boost-date-time \
- boost-format \
- boost-locale \
- boost-multiprecision \
- boost-predef \
- boost-regex \
- boost-system \
- boost-variant \
- # Use enable rtti to avoid link problems in Gandiva
- llvm[clang,default-options,default-targets,lld,tools,enable-rtti]
+# Use enable llvm[enable-rtti] in the vcpkg.json to avoid link problems in Gandiva
+RUN vcpkg install \
+ --clean-after-build \
+ --x-install-root=${VCPKG_ROOT}/installed \
+ --x-manifest-root=/arrow/ci/vcpkg \
+ --x-feature=flight \
+ --x-feature=gcs \
+ --x-feature=json \
+ --x-feature=parquet \
+ --x-feature=gandiva
 
 # Install Java
 ARG java=1.8.0

ci/docker/python-wheel-manylinux-201x.dockerfile

@@ -53,49 +53,35 @@ COPY ci/vcpkg/*.patch \
 COPY ci/scripts/install_vcpkg.sh \
  ci/scripts/install_glibc.sh \
  arrow/ci/scripts/
-RUN arrow/ci/scripts/install_vcpkg.sh /opt/vcpkg ${vcpkg} && \
+ENV VCPKG_ROOT=/opt/vcpkg
+RUN arrow/ci/scripts/install_vcpkg.sh ${VCPKG_ROOT} ${vcpkg} && \
  if [ "${manylinux}" == "2010" ]; then \
  arrow/ci/scripts/install_glibc.sh ${glibc} /opt/glibc-${glibc} && \
- patchelf --set-interpreter /opt/glibc-2.18/lib/ld-linux-x86-64.so.2 /opt/vcpkg/vcpkg && \
- patchelf --set-rpath /opt/glibc-2.18/lib:/usr/lib64 /opt/vcpkg/vcpkg; \
+ patchelf --set-interpreter /opt/glibc-2.18/lib/ld-linux-x86-64.so.2 ${VCPKG_ROOT}/vcpkg && \
+ patchelf --set-rpath /opt/glibc-2.18/lib:/usr/lib64 ${VCPKG_ROOT}/vcpkg; \
  fi
-ENV PATH="/opt/vcpkg:${PATH}"
+ENV PATH="${PATH}:${VCPKG_ROOT}"
 
 ARG build_type=release
 ENV CMAKE_BUILD_TYPE=${build_type} \
  VCPKG_FORCE_SYSTEM_BINARIES=1 \
  VCPKG_OVERLAY_TRIPLETS=/arrow/ci/vcpkg \
  VCPKG_DEFAULT_TRIPLET=${arch_short}-linux-static-${build_type} \
- VCPKG_FEATURE_FLAGS=-manifests
-
-# Need to install the boost-build prior installing the boost packages, otherwise
-# vcpkg will raise an error.
-# TODO(kszucs): factor out the package enumeration to a text file and reuse it
-# from the windows image and potentially in a future macos wheel build
-RUN vcpkg install --clean-after-build \
- abseil \
- aws-sdk-cpp[config,cognito-identity,core,identity-management,s3,sts,transfer] \
- boost-filesystem \
- brotli \
- bzip2 \
- c-ares \
- curl \
- flatbuffers \
- gflags \
- glog \
- google-cloud-cpp[core,storage] \
- grpc \
- lz4 \
- openssl \
- orc \
- protobuf \
- rapidjson \
- re2 \
- snappy \
- thrift \
- utf8proc \
- zlib \
- zstd
+ VCPKG_FEATURE_FLAGS="manifests"
+COPY ci/vcpkg/vcpkg.json arrow/ci/vcpkg/
+# cannot use the S3 feature here because while aws-sdk-cpp=1.9.160 contains
+# ssl related fixies as well as we can patch the vcpkg portfile to support
+# arm machines it hits ARROW-15141 where we would need to fall back to 1.8.186
+# but we cannot patch those portfiles since vcpkg-tool handles the checkout of
+# previous versions => use bundled S3 build
+RUN vcpkg install \
+ --clean-after-build \
+ --x-install-root=${VCPKG_ROOT}/installed \
+ --x-manifest-root=/arrow/ci/vcpkg \
+ --x-feature=flight \
+ --x-feature=gcs \
+ --x-feature=json \
+ --x-feature=parquet
 
 ARG python=3.8
 ENV PYTHON_VERSION=${python}

ci/docker/python-wheel-windows-vs2017.dockerfile

@@ -36,8 +36,9 @@ COPY ci/vcpkg/*.patch \
  ci/vcpkg/*windows*.cmake \
  arrow/ci/vcpkg/
 COPY ci/scripts/install_vcpkg.sh arrow/ci/scripts/
+ENV VCPKG_ROOT=C:\\vcpkg
 RUN bash arrow/ci/scripts/install_vcpkg.sh /c/vcpkg %vcpkg% && \
- setx PATH "%PATH%;C:\vcpkg"
+ setx PATH "%PATH%;%VCPKG_ROOT%"
 
 # Configure vcpkg and install dependencies
 # NOTE: use windows batch environment notation for build arguments in RUN
@@ -48,34 +49,21 @@ ARG build_type=release
 ENV CMAKE_BUILD_TYPE=${build_type} \
  VCPKG_OVERLAY_TRIPLETS=C:\\arrow\\ci\\vcpkg \
  VCPKG_DEFAULT_TRIPLET=amd64-windows-static-md-${build_type} \
- VCPKG_FEATURE_FLAGS=-manifests
-
-RUN vcpkg install --clean-after-build \
- abseil \
- aws-sdk-cpp[config,cognito-identity,core,identity-management,s3,sts,transfer] \
- boost-filesystem \
- boost-multiprecision \
- boost-system \
- brotli \
- bzip2 \
- c-ares \
- curl \
- flatbuffers \
- gflags \
- glog \
- google-cloud-cpp[core,storage] \
- grpc \
- lz4 \
- openssl \
- orc \
- protobuf \
- rapidjson \
- re2 \
- snappy \
- thrift \
- utf8proc \
- zlib \
- zstd
+ VCPKG_FEATURE_FLAGS="manifests"
+COPY ci/vcpkg/vcpkg.json arrow/ci/vcpkg/
+# cannot use the S3 feature here because while aws-sdk-cpp=1.9.160 contains
+# ssl related fixies as well as we can patch the vcpkg portfile to support
+# arm machines it hits ARROW-15141 where we would need to fall back to 1.8.186
+# but we cannot patch those portfiles since vcpkg-tool handles the checkout of
+# previous versions => use bundled S3 build
+RUN vcpkg install \
+ --clean-after-build \
+ --x-install-root=%VCPKG_ROOT%\installed \
+ --x-manifest-root=arrow/ci/vcpkg \
+ --x-feature=flight \
+ --x-feature=gcs \
+ --x-feature=json \
+ --x-feature=parquet
 
 # Remove previous installations of python from the base image
 # NOTE: a more recent base image (tried with 2.12.1) comes with python 3.9.7
@@ -87,20 +75,20 @@ RUN wmic product where "name like 'python%%'" call uninstall /nointeractive && \
 
 # Define the full version number otherwise choco falls back to patch number 0 (3.7 => 3.7.0)
 ARG python=3.8
-RUN (if "%python%"=="3.7" setx PYTHON_VERSION 3.7.12) & \
- (if "%python%"=="3.8" setx PYTHON_VERSION 3.8.11) & \
- (if "%python%"=="3.9" setx PYTHON_VERSION 3.9.9) & \
- (if "%python%"=="3.10" setx PYTHON_VERSION 3.10.1)
+RUN (if "%python%"=="3.7" setx PYTHON_VERSION "3.7.9" && setx PATH "%PATH%;C:\Python37;C:\Python37\Scripts") & \
+ (if "%python%"=="3.8" setx PYTHON_VERSION "3.8.10" && setx PATH "%PATH%;C:\Python38;C:\Python38\Scripts") & \
+ (if "%python%"=="3.9" setx PYTHON_VERSION "3.9.7" && setx PATH "%PATH%;C:\Python39;C:\Python39\Scripts") & \
+ (if "%python%"=="3.10" setx PYTHON_VERSION "3.10.2" && setx PATH "%PATH%;C:\Python310;C:\Python310\Scripts")
 RUN choco install -r -y --no-progress python --version=%PYTHON_VERSION%
-RUN pip install -U pip
+RUN python -m pip install -U pip setuptools
 
 COPY python/requirements-wheel-build.txt arrow/python/
-RUN pip install -r arrow/python/requirements-wheel-build.txt
+RUN python -m pip install -r arrow/python/requirements-wheel-build.txt
 
-ENV CLCACHE_DIR="C:\clcache"
-ENV CLCACHE_COMPRESS=1
-ENV CLCACHE_COMPRESSLEVEL=6
-RUN pip install git+https://github.com/Nuitka/clcache.git
+# ENV CLCACHE_DIR="C:\clcache"
+# ENV CLCACHE_COMPRESS=1
+# ENV CLCACHE_COMPRESSLEVEL=6
+# RUN pip install git+https://github.com/Nuitka/clcache.git
 
 # For debugging purposes
 # RUN wget --no-check-certificate https://github.com/lucasg/Dependencies/releases/download/v1.10/Dependencies_x64_Release.zip

ci/scripts/java_jni_manylinux_build.sh

@@ -55,9 +55,6 @@ export ARROW_TEST_DATA="${arrow_dir}/testing/data"
 export PARQUET_TEST_DATA="${arrow_dir}/cpp/submodules/parquet-testing/data"
 export AWS_EC2_METADATA_DISABLED=TRUE
 
-# NOTE(kszucs): workaround for ARROW-15403 along with the ORC_* cmake variables
-vcpkg remove orc
-
 mkdir -p "${build_dir}"
 pushd "${build_dir}"
 

ci/scripts/python_wheel_macos_build.sh

@@ -87,16 +87,11 @@ echo "=== (${PYTHON_VERSION}) Building Arrow C++ libraries ==="
 : ${VCPKG_FEATURE_FLAGS:=-manifests}
 : ${VCPKG_TARGET_TRIPLET:=${VCPKG_DEFAULT_TRIPLET:-x64-osx-static-${CMAKE_BUILD_TYPE}}}
 
-# NOTE(kszucs): workaround for ARROW-15403 along with the ORC_* cmake variables
-vcpkg remove orc
-
 mkdir -p ${build_dir}/build
 pushd ${build_dir}/build
 
 cmake \
  -DARROW_BUILD_SHARED=ON \
- -DCMAKE_APPLE_SILICON_PROCESSOR=arm64 \
- -DCMAKE_OSX_ARCHITECTURES=${CMAKE_OSX_ARCHITECTURES} \
  -DARROW_BUILD_STATIC=OFF \
  -DARROW_BUILD_TESTS=OFF \
  -DARROW_DATASET=${ARROW_DATASET} \
@@ -124,13 +119,16 @@ cmake \
  -DARROW_WITH_SNAPPY=${ARROW_WITH_SNAPPY} \
  -DARROW_WITH_ZLIB=${ARROW_WITH_ZLIB} \
  -DARROW_WITH_ZSTD=${ARROW_WITH_ZSTD} \
+ -DAWSSDK_SOURCE=BUNDLED \
+ -DCMAKE_APPLE_SILICON_PROCESSOR=arm64 \
  -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} \
  -DCMAKE_INSTALL_LIBDIR=lib \
  -DCMAKE_INSTALL_PREFIX=${build_dir}/install \
+ -DCMAKE_OSX_ARCHITECTURES=${CMAKE_OSX_ARCHITECTURES} \
  -DCMAKE_UNITY_BUILD=${CMAKE_UNITY_BUILD} \
  -DOPENSSL_USE_STATIC_LIBS=ON \
- -DORC_SOURCE=BUNDLED \
  -DORC_PROTOBUF_EXECUTABLE=${VCPKG_ROOT}/installed/${VCPKG_TARGET_TRIPLET}/tools/protobuf/protoc \
+ -DORC_SOURCE=BUNDLED \
  -DVCPKG_MANIFEST_MODE=OFF \
  -DVCPKG_TARGET_TRIPLET=${VCPKG_TARGET_TRIPLET} \
  -G ${CMAKE_GENERATOR} \

ci/scripts/python_wheel_manylinux_build.sh

@@ -80,11 +80,9 @@ if [[ "$(uname -m)" == arm* ]] || [[ "$(uname -m)" == aarch* ]]; then
  export ARROW_EXTRA_CMAKE_FLAGS="-DARROW_JEMALLOC_LG_PAGE=16"
 fi
 
-# NOTE(kszucs): workaround for ARROW-15403 along with the ORC_* cmake variables
-vcpkg remove orc
-
 mkdir /tmp/arrow-build
 pushd /tmp/arrow-build
+
 cmake \
  -DARROW_BROTLI_USE_SHARED=OFF \
  -DARROW_BUILD_SHARED=ON \
@@ -115,13 +113,14 @@ cmake \
  -DARROW_WITH_SNAPPY=${ARROW_WITH_SNAPPY} \
  -DARROW_WITH_ZLIB=${ARROW_WITH_ZLIB} \
  -DARROW_WITH_ZSTD=${ARROW_WITH_ZSTD} \
+ -DAWSSDK_SOURCE=BUNDLED \
  -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} \
  -DCMAKE_INSTALL_LIBDIR=lib \
  -DCMAKE_INSTALL_PREFIX=/tmp/arrow-dist \
  -DCMAKE_UNITY_BUILD=${CMAKE_UNITY_BUILD} \
  -DOPENSSL_USE_STATIC_LIBS=ON \
- -DORC_SOURCE=BUNDLED \
  -DORC_PROTOBUF_EXECUTABLE=${VCPKG_ROOT}/installed/${VCPKG_TARGET_TRIPLET}/tools/protobuf/protoc \
+ -DORC_SOURCE=BUNDLED \
  -DVCPKG_MANIFEST_MODE=OFF \
  -DVCPKG_TARGET_TRIPLET=${VCPKG_TARGET_TRIPLET} \
  ${ARROW_EXTRA_CMAKE_FLAGS} \

ci/scripts/python_wheel_windows_build.bat

@@ -51,9 +51,6 @@ set VCPKG_ROOT=C:\vcpkg
 set VCPKG_FEATURE_FLAGS=-manifests
 set VCGPK_TARGET_TRIPLET=amd64-windows-static-md-%CMAKE_BUILD_TYPE%
 
-@rem NOTE(kszucs): workaround for ARROW-15403 along with the ORC_* cmake variables
-vcpkg remove orc
-
 mkdir C:\arrow-build
 pushd C:\arrow-build
 cmake ^
@@ -80,13 +77,12 @@ cmake ^
  -DARROW_WITH_SNAPPY=%ARROW_WITH_SNAPPY% ^
  -DARROW_WITH_ZLIB=%ARROW_WITH_ZLIB% ^
  -DARROW_WITH_ZSTD=%ARROW_WITH_ZSTD% ^
+ -DAWSSDK_SOURCE=BUNDLED ^
  -DCMAKE_BUILD_TYPE=%CMAKE_BUILD_TYPE% ^
  -DCMAKE_CXX_COMPILER=clcache ^
  -DCMAKE_INSTALL_PREFIX=C:\arrow-dist ^
  -DCMAKE_UNITY_BUILD=%CMAKE_UNITY_BUILD% ^
  -DMSVC_LINK_VERBOSE=ON ^
- -DORC_SOURCE=BUNDLED ^
- -DORC_PROTOBUF_EXECUTABLE=%VCPKG_ROOT%\installed\%VCGPK_TARGET_TRIPLET%\tools\protobuf\protoc.exe ^
  -DVCPKG_MANIFEST_MODE=OFF ^
  -DVCPKG_TARGET_TRIPLET=%VCGPK_TARGET_TRIPLET% ^
  -G "%CMAKE_GENERATOR%" ^

ci/scripts/python_wheel_windows_test.bat

@@ -41,15 +41,15 @@ pip install -r C:\arrow\python\requirements-wheel-test.txt || exit /B
 python -m pip install --no-index --find-links=C:\arrow\python\dist\ pyarrow || exit /B
 
 @REM Test that the modules are importable
-python -c "import pyarrow"
-python -c "import pyarrow._hdfs"
-python -c "import pyarrow._s3fs"
-python -c "import pyarrow.csv"
-python -c "import pyarrow.dataset"
-python -c "import pyarrow.flight"
-python -c "import pyarrow.fs"
-python -c "import pyarrow.json"
-python -c "import pyarrow.parquet"
+python -c "import pyarrow" || exit /B
+python -c "import pyarrow._hdfs" || exit /B
+python -c "import pyarrow._s3fs" || exit /B
+python -c "import pyarrow.csv" || exit /B
+python -c "import pyarrow.dataset" || exit /B
+python -c "import pyarrow.flight" || exit /B
+python -c "import pyarrow.fs" || exit /B
+python -c "import pyarrow.json" || exit /B
+python -c "import pyarrow.parquet" || exit /B
 
 @REM Execute unittest
 pytest -r s --pyargs pyarrow || exit /B

ci/vcpkg/ports.patch

@@ -164,20 +164,20 @@ index 00000000..4566cc66
 +- " set(LibCrypto_SHARED_LIBRARY ${OPENSSL_CRYPTO_LIBRARY} CACHE INTERNAL \"The OpenSSL crypto shared library\")\n"
 +- "endif()\n"
 + )
-+
++
 + if (NOT SIMPLE_INSTALL)
 diff --git a/ports/aws-sdk-cpp/portfile.cmake b/ports/aws-sdk-cpp/portfile.cmake
 index 2d6bba4d..0ac47887 100644
 --- a/ports/aws-sdk-cpp/portfile.cmake
 +++ b/ports/aws-sdk-cpp/portfile.cmake
 @@ -8,6 +8,7 @@ vcpkg_from_github(
- PATCHES
- patch-relocatable-rpath.patch
- fix-aws-root.patch
-+ fix-find-crypto.patch
- )
-
- string(COMPARE EQUAL "${VCPKG_CRT_LINKAGE}" "dynamic" FORCE_SHARED_CRT)
+ PATCHES
+ patch-relocatable-rpath.patch
+ fix-aws-root.patch
++ fix-find-crypto.patch
+ )
+
+ string(COMPARE EQUAL "${VCPKG_CRT_LINKAGE}" "dynamic" FORCE_SHARED_CRT)
 diff --git a/ports/aws-sdk-cpp/vcpkg.json b/ports/aws-sdk-cpp/vcpkg.json
 index 3836e2b6..cd88ef07 100644
 --- a/ports/aws-sdk-cpp/vcpkg.json
@@ -216,7 +216,7 @@ index 45b8c706..b409d8a7 100644
  HEAD_REF master
 + PATCHES "snappy-disable-bmi.patch"
  )
-
+
  vcpkg_cmake_configure(
 diff --git a/ports/snappy/snappy-disable-bmi.patch b/ports/snappy/snappy-disable-bmi.patch
 new file mode 100644
@@ -241,5 +241,5 @@ index 00000000..a57ce0c2
 + return v & ~(mask << (8 * n));
 +-#endif
 + }
-+
++
 + static inline bool LeftShiftOverflows(uint8_t value, uint32_t shift) {

ci/docker/java-jni-manylinux-201x.dockerfile

@@ -19,19 +19,16 @@ ARG base
 FROM ${base}
 
 # Install the libaries required by the Gandiva to run
-RUN vcpkg install --clean-after-build \
- boost-algorithm \
- boost-crc \
- boost-date-time \
- boost-format \
- boost-locale \
- boost-multiprecision \
- boost-predef \
- boost-regex \
- boost-system \
- boost-variant \
- # Use enable rtti to avoid link problems in Gandiva
- llvm[clang,default-options,default-targets,lld,tools,enable-rtti]
+# Use enable llvm[enable-rtti] in the vcpkg.json to avoid link problems in Gandiva
+RUN vcpkg install \
+ --clean-after-build \
+ --x-install-root=${VCPKG_ROOT}/installed \
+ --x-manifest-root=/arrow/ci/vcpkg \
+ --x-feature=flight \
+ --x-feature=gcs \
+ --x-feature=json \
+ --x-feature=parquet \
+ --x-feature=gandiva
 
 # Install Java
 ARG java=1.8.0

ci/docker/python-wheel-manylinux-201x.dockerfile

@@ -53,49 +53,35 @@ COPY ci/vcpkg/*.patch \
 COPY ci/scripts/install_vcpkg.sh \
  ci/scripts/install_glibc.sh \
  arrow/ci/scripts/
-RUN arrow/ci/scripts/install_vcpkg.sh /opt/vcpkg ${vcpkg} && \
+ENV VCPKG_ROOT=/opt/vcpkg
+RUN arrow/ci/scripts/install_vcpkg.sh ${VCPKG_ROOT} ${vcpkg} && \
  if [ "${manylinux}" == "2010" ]; then \
  arrow/ci/scripts/install_glibc.sh ${glibc} /opt/glibc-${glibc} && \
- patchelf --set-interpreter /opt/glibc-2.18/lib/ld-linux-x86-64.so.2 /opt/vcpkg/vcpkg && \
- patchelf --set-rpath /opt/glibc-2.18/lib:/usr/lib64 /opt/vcpkg/vcpkg; \
+ patchelf --set-interpreter /opt/glibc-2.18/lib/ld-linux-x86-64.so.2 ${VCPKG_ROOT}/vcpkg && \
+ patchelf --set-rpath /opt/glibc-2.18/lib:/usr/lib64 ${VCPKG_ROOT}/vcpkg; \
  fi
-ENV PATH="/opt/vcpkg:${PATH}"
+ENV PATH="${PATH}:${VCPKG_ROOT}"
 
 ARG build_type=release
 ENV CMAKE_BUILD_TYPE=${build_type} \
  VCPKG_FORCE_SYSTEM_BINARIES=1 \
  VCPKG_OVERLAY_TRIPLETS=/arrow/ci/vcpkg \
  VCPKG_DEFAULT_TRIPLET=${arch_short}-linux-static-${build_type} \
- VCPKG_FEATURE_FLAGS=-manifests
-
-# Need to install the boost-build prior installing the boost packages, otherwise
-# vcpkg will raise an error.
-# TODO(kszucs): factor out the package enumeration to a text file and reuse it
-# from the windows image and potentially in a future macos wheel build
-RUN vcpkg install --clean-after-build \
- abseil \
- aws-sdk-cpp[config,cognito-identity,core,identity-management,s3,sts,transfer] \
- boost-filesystem \
- brotli \
- bzip2 \
- c-ares \
- curl \
- flatbuffers \
- gflags \
- glog \
- google-cloud-cpp[core,storage] \
- grpc \
- lz4 \
- openssl \
- orc \
- protobuf \
- rapidjson \
- re2 \
- snappy \
- thrift \
- utf8proc \
- zlib \
- zstd
+ VCPKG_FEATURE_FLAGS="manifests"
+COPY ci/vcpkg/vcpkg.json arrow/ci/vcpkg/
+# cannot use the S3 feature here because while aws-sdk-cpp=1.9.160 contains
+# ssl related fixies as well as we can patch the vcpkg portfile to support
+# arm machines it hits ARROW-15141 where we would need to fall back to 1.8.186
+# but we cannot patch those portfiles since vcpkg-tool handles the checkout of
+# previous versions => use bundled S3 build
+RUN vcpkg install \
+ --clean-after-build \
+ --x-install-root=${VCPKG_ROOT}/installed \
+ --x-manifest-root=/arrow/ci/vcpkg \
+ --x-feature=flight \
+ --x-feature=gcs \
+ --x-feature=json \
+ --x-feature=parquet
 
 ARG python=3.8
 ENV PYTHON_VERSION=${python}

ci/docker/python-wheel-windows-vs2017.dockerfile

@@ -36,8 +36,9 @@ COPY ci/vcpkg/*.patch \
  ci/vcpkg/*windows*.cmake \
  arrow/ci/vcpkg/
 COPY ci/scripts/install_vcpkg.sh arrow/ci/scripts/
+ENV VCPKG_ROOT=C:\\vcpkg
 RUN bash arrow/ci/scripts/install_vcpkg.sh /c/vcpkg %vcpkg% && \
- setx PATH "%PATH%;C:\vcpkg"
+ setx PATH "%PATH%;%VCPKG_ROOT%"
 
 # Configure vcpkg and install dependencies
 # NOTE: use windows batch environment notation for build arguments in RUN
@@ -48,34 +49,21 @@ ARG build_type=release
 ENV CMAKE_BUILD_TYPE=${build_type} \
  VCPKG_OVERLAY_TRIPLETS=C:\\arrow\\ci\\vcpkg \
  VCPKG_DEFAULT_TRIPLET=amd64-windows-static-md-${build_type} \
- VCPKG_FEATURE_FLAGS=-manifests
-
-RUN vcpkg install --clean-after-build \
- abseil \
- aws-sdk-cpp[config,cognito-identity,core,identity-management,s3,sts,transfer] \
- boost-filesystem \
- boost-multiprecision \
- boost-system \
- brotli \
- bzip2 \
- c-ares \
- curl \
- flatbuffers \
- gflags \
- glog \
- google-cloud-cpp[core,storage] \
- grpc \
- lz4 \
- openssl \
- orc \
- protobuf \
- rapidjson \
- re2 \
- snappy \
- thrift \
- utf8proc \
- zlib \
- zstd
+ VCPKG_FEATURE_FLAGS="manifests"
+COPY ci/vcpkg/vcpkg.json arrow/ci/vcpkg/
+# cannot use the S3 feature here because while aws-sdk-cpp=1.9.160 contains
+# ssl related fixies as well as we can patch the vcpkg portfile to support
+# arm machines it hits ARROW-15141 where we would need to fall back to 1.8.186
+# but we cannot patch those portfiles since vcpkg-tool handles the checkout of
+# previous versions => use bundled S3 build
+RUN vcpkg install \
+ --clean-after-build \
+ --x-install-root=%VCPKG_ROOT%\installed \
+ --x-manifest-root=arrow/ci/vcpkg \
+ --x-feature=flight \
+ --x-feature=gcs \
+ --x-feature=json \
+ --x-feature=parquet
 
 # Remove previous installations of python from the base image
 # NOTE: a more recent base image (tried with 2.12.1) comes with python 3.9.7
@@ -87,20 +75,20 @@ RUN wmic product where "name like 'python%%'" call uninstall /nointeractive && \
 
 # Define the full version number otherwise choco falls back to patch number 0 (3.7 => 3.7.0)
 ARG python=3.8
-RUN (if "%python%"=="3.7" setx PYTHON_VERSION 3.7.12) & \
- (if "%python%"=="3.8" setx PYTHON_VERSION 3.8.11) & \
- (if "%python%"=="3.9" setx PYTHON_VERSION 3.9.9) & \
- (if "%python%"=="3.10" setx PYTHON_VERSION 3.10.1)
+RUN (if "%python%"=="3.7" setx PYTHON_VERSION "3.7.9" && setx PATH "%PATH%;C:\Python37;C:\Python37\Scripts") & \
+ (if "%python%"=="3.8" setx PYTHON_VERSION "3.8.10" && setx PATH "%PATH%;C:\Python38;C:\Python38\Scripts") & \
+ (if "%python%"=="3.9" setx PYTHON_VERSION "3.9.7" && setx PATH "%PATH%;C:\Python39;C:\Python39\Scripts") & \
+ (if "%python%"=="3.10" setx PYTHON_VERSION "3.10.2" && setx PATH "%PATH%;C:\Python310;C:\Python310\Scripts")
 RUN choco install -r -y --no-progress python --version=%PYTHON_VERSION%
-RUN pip install -U pip
+RUN python -m pip install -U pip setuptools
 
 COPY python/requirements-wheel-build.txt arrow/python/
-RUN pip install -r arrow/python/requirements-wheel-build.txt
+RUN python -m pip install -r arrow/python/requirements-wheel-build.txt
 
-ENV CLCACHE_DIR="C:\clcache"
-ENV CLCACHE_COMPRESS=1
-ENV CLCACHE_COMPRESSLEVEL=6
-RUN pip install git+https://github.com/Nuitka/clcache.git
+# ENV CLCACHE_DIR="C:\clcache"
+# ENV CLCACHE_COMPRESS=1
+# ENV CLCACHE_COMPRESSLEVEL=6
+# RUN pip install git+https://github.com/Nuitka/clcache.git
 
 # For debugging purposes
 # RUN wget --no-check-certificate https://github.com/lucasg/Dependencies/releases/download/v1.10/Dependencies_x64_Release.zip

ci/scripts/java_jni_manylinux_build.sh

@@ -55,9 +55,6 @@ export ARROW_TEST_DATA="${arrow_dir}/testing/data"
 export PARQUET_TEST_DATA="${arrow_dir}/cpp/submodules/parquet-testing/data"
 export AWS_EC2_METADATA_DISABLED=TRUE
 
-# NOTE(kszucs): workaround for ARROW-15403 along with the ORC_* cmake variables
-vcpkg remove orc
-
 mkdir -p "${build_dir}"
 pushd "${build_dir}"
 

ci/scripts/python_wheel_macos_build.sh

@@ -87,16 +87,11 @@ echo "=== (${PYTHON_VERSION}) Building Arrow C++ libraries ==="
 : ${VCPKG_FEATURE_FLAGS:=-manifests}
 : ${VCPKG_TARGET_TRIPLET:=${VCPKG_DEFAULT_TRIPLET:-x64-osx-static-${CMAKE_BUILD_TYPE}}}
 
-# NOTE(kszucs): workaround for ARROW-15403 along with the ORC_* cmake variables
-vcpkg remove orc
-
 mkdir -p ${build_dir}/build
 pushd ${build_dir}/build
 
 cmake \
  -DARROW_BUILD_SHARED=ON \
- -DCMAKE_APPLE_SILICON_PROCESSOR=arm64 \
- -DCMAKE_OSX_ARCHITECTURES=${CMAKE_OSX_ARCHITECTURES} \
  -DARROW_BUILD_STATIC=OFF \
  -DARROW_BUILD_TESTS=OFF \
  -DARROW_DATASET=${ARROW_DATASET} \
@@ -124,13 +119,16 @@ cmake \
  -DARROW_WITH_SNAPPY=${ARROW_WITH_SNAPPY} \
  -DARROW_WITH_ZLIB=${ARROW_WITH_ZLIB} \
  -DARROW_WITH_ZSTD=${ARROW_WITH_ZSTD} \
+ -DAWSSDK_SOURCE=BUNDLED \
+ -DCMAKE_APPLE_SILICON_PROCESSOR=arm64 \
  -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} \
  -DCMAKE_INSTALL_LIBDIR=lib \
  -DCMAKE_INSTALL_PREFIX=${build_dir}/install \
+ -DCMAKE_OSX_ARCHITECTURES=${CMAKE_OSX_ARCHITECTURES} \
  -DCMAKE_UNITY_BUILD=${CMAKE_UNITY_BUILD} \
  -DOPENSSL_USE_STATIC_LIBS=ON \
- -DORC_SOURCE=BUNDLED \
  -DORC_PROTOBUF_EXECUTABLE=${VCPKG_ROOT}/installed/${VCPKG_TARGET_TRIPLET}/tools/protobuf/protoc \
+ -DORC_SOURCE=BUNDLED \
  -DVCPKG_MANIFEST_MODE=OFF \
  -DVCPKG_TARGET_TRIPLET=${VCPKG_TARGET_TRIPLET} \
  -G ${CMAKE_GENERATOR} \

ci/scripts/python_wheel_manylinux_build.sh

@@ -80,11 +80,9 @@ if [[ "$(uname -m)" == arm* ]] || [[ "$(uname -m)" == aarch* ]]; then
  export ARROW_EXTRA_CMAKE_FLAGS="-DARROW_JEMALLOC_LG_PAGE=16"
 fi
 
-# NOTE(kszucs): workaround for ARROW-15403 along with the ORC_* cmake variables
-vcpkg remove orc
-
 mkdir /tmp/arrow-build
 pushd /tmp/arrow-build
+
 cmake \
  -DARROW_BROTLI_USE_SHARED=OFF \
  -DARROW_BUILD_SHARED=ON \
@@ -115,13 +113,14 @@ cmake \
  -DARROW_WITH_SNAPPY=${ARROW_WITH_SNAPPY} \
  -DARROW_WITH_ZLIB=${ARROW_WITH_ZLIB} \
  -DARROW_WITH_ZSTD=${ARROW_WITH_ZSTD} \
+ -DAWSSDK_SOURCE=BUNDLED \
  -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} \
  -DCMAKE_INSTALL_LIBDIR=lib \
  -DCMAKE_INSTALL_PREFIX=/tmp/arrow-dist \
  -DCMAKE_UNITY_BUILD=${CMAKE_UNITY_BUILD} \
  -DOPENSSL_USE_STATIC_LIBS=ON \
- -DORC_SOURCE=BUNDLED \
  -DORC_PROTOBUF_EXECUTABLE=${VCPKG_ROOT}/installed/${VCPKG_TARGET_TRIPLET}/tools/protobuf/protoc \
+ -DORC_SOURCE=BUNDLED \
  -DVCPKG_MANIFEST_MODE=OFF \
  -DVCPKG_TARGET_TRIPLET=${VCPKG_TARGET_TRIPLET} \
  ${ARROW_EXTRA_CMAKE_FLAGS} \

ci/scripts/python_wheel_windows_build.bat

@@ -51,9 +51,6 @@ set VCPKG_ROOT=C:\vcpkg
 set VCPKG_FEATURE_FLAGS=-manifests
 set VCGPK_TARGET_TRIPLET=amd64-windows-static-md-%CMAKE_BUILD_TYPE%
 
-@rem NOTE(kszucs): workaround for ARROW-15403 along with the ORC_* cmake variables
-vcpkg remove orc
-
 mkdir C:\arrow-build
 pushd C:\arrow-build
 cmake ^
@@ -80,13 +77,12 @@ cmake ^
  -DARROW_WITH_SNAPPY=%ARROW_WITH_SNAPPY% ^
  -DARROW_WITH_ZLIB=%ARROW_WITH_ZLIB% ^
  -DARROW_WITH_ZSTD=%ARROW_WITH_ZSTD% ^
+ -DAWSSDK_SOURCE=BUNDLED ^
  -DCMAKE_BUILD_TYPE=%CMAKE_BUILD_TYPE% ^
  -DCMAKE_CXX_COMPILER=clcache ^
  -DCMAKE_INSTALL_PREFIX=C:\arrow-dist ^
  -DCMAKE_UNITY_BUILD=%CMAKE_UNITY_BUILD% ^
  -DMSVC_LINK_VERBOSE=ON ^
- -DORC_SOURCE=BUNDLED ^
- -DORC_PROTOBUF_EXECUTABLE=%VCPKG_ROOT%\installed\%VCGPK_TARGET_TRIPLET%\tools\protobuf\protoc.exe ^
  -DVCPKG_MANIFEST_MODE=OFF ^
  -DVCPKG_TARGET_TRIPLET=%VCGPK_TARGET_TRIPLET% ^
  -G "%CMAKE_GENERATOR%" ^

ci/scripts/python_wheel_windows_test.bat

@@ -41,15 +41,15 @@ pip install -r C:\arrow\python\requirements-wheel-test.txt || exit /B
 python -m pip install --no-index --find-links=C:\arrow\python\dist\ pyarrow || exit /B
 
 @REM Test that the modules are importable
-python -c "import pyarrow"
-python -c "import pyarrow._hdfs"
-python -c "import pyarrow._s3fs"
-python -c "import pyarrow.csv"
-python -c "import pyarrow.dataset"
-python -c "import pyarrow.flight"
-python -c "import pyarrow.fs"
-python -c "import pyarrow.json"
-python -c "import pyarrow.parquet"
+python -c "import pyarrow" || exit /B
+python -c "import pyarrow._hdfs" || exit /B
+python -c "import pyarrow._s3fs" || exit /B
+python -c "import pyarrow.csv" || exit /B
+python -c "import pyarrow.dataset" || exit /B
+python -c "import pyarrow.flight" || exit /B
+python -c "import pyarrow.fs" || exit /B
+python -c "import pyarrow.json" || exit /B
+python -c "import pyarrow.parquet" || exit /B
 
 @REM Execute unittest
 pytest -r s --pyargs pyarrow || exit /B

ci/vcpkg/ports.patch

@@ -164,20 +164,20 @@ index 00000000..4566cc66
 +- " set(LibCrypto_SHARED_LIBRARY ${OPENSSL_CRYPTO_LIBRARY} CACHE INTERNAL \"The OpenSSL crypto shared library\")\n"
 +- "endif()\n"
 + )
-+
++
 + if (NOT SIMPLE_INSTALL)
 diff --git a/ports/aws-sdk-cpp/portfile.cmake b/ports/aws-sdk-cpp/portfile.cmake
 index 2d6bba4d..0ac47887 100644
 --- a/ports/aws-sdk-cpp/portfile.cmake
 +++ b/ports/aws-sdk-cpp/portfile.cmake
 @@ -8,6 +8,7 @@ vcpkg_from_github(
- PATCHES
- patch-relocatable-rpath.patch
- fix-aws-root.patch
-+ fix-find-crypto.patch
- )
-
- string(COMPARE EQUAL "${VCPKG_CRT_LINKAGE}" "dynamic" FORCE_SHARED_CRT)
+ PATCHES
+ patch-relocatable-rpath.patch
+ fix-aws-root.patch
++ fix-find-crypto.patch
+ )
+
+ string(COMPARE EQUAL "${VCPKG_CRT_LINKAGE}" "dynamic" FORCE_SHARED_CRT)
 diff --git a/ports/aws-sdk-cpp/vcpkg.json b/ports/aws-sdk-cpp/vcpkg.json
 index 3836e2b6..cd88ef07 100644
 --- a/ports/aws-sdk-cpp/vcpkg.json
@@ -216,7 +216,7 @@ index 45b8c706..b409d8a7 100644
  HEAD_REF master
 + PATCHES "snappy-disable-bmi.patch"
  )
-
+
  vcpkg_cmake_configure(
 diff --git a/ports/snappy/snappy-disable-bmi.patch b/ports/snappy/snappy-disable-bmi.patch
 new file mode 100644
@@ -241,5 +241,5 @@ index 00000000..a57ce0c2
 + return v & ~(mask << (8 * n));
 +-#endif
 + }
-+
++
 + static inline bool LeftShiftOverflows(uint8_t value, uint32_t shift) {

ci/docker/java-jni-manylinux-201x.dockerfile

@@ -19,19 +19,16 @@ ARG base
 FROM ${base}
 
 # Install the libaries required by the Gandiva to run
-RUN vcpkg install --clean-after-build \
- boost-algorithm \
- boost-crc \
- boost-date-time \
- boost-format \
- boost-locale \
- boost-multiprecision \
- boost-predef \
- boost-regex \
- boost-system \
- boost-variant \
- # Use enable rtti to avoid link problems in Gandiva
- llvm[clang,default-options,default-targets,lld,tools,enable-rtti]
+# Use enable llvm[enable-rtti] in the vcpkg.json to avoid link problems in Gandiva
+RUN vcpkg install \
+ --clean-after-build \
+ --x-install-root=${VCPKG_ROOT}/installed \
+ --x-manifest-root=/arrow/ci/vcpkg \
+ --x-feature=flight \
+ --x-feature=gcs \
+ --x-feature=json \
+ --x-feature=parquet \
+ --x-feature=gandiva
 
 # Install Java
 ARG java=1.8.0

ci/docker/python-wheel-manylinux-201x.dockerfile

@@ -53,49 +53,35 @@ COPY ci/vcpkg/*.patch \
 COPY ci/scripts/install_vcpkg.sh \
  ci/scripts/install_glibc.sh \
  arrow/ci/scripts/
-RUN arrow/ci/scripts/install_vcpkg.sh /opt/vcpkg ${vcpkg} && \
+ENV VCPKG_ROOT=/opt/vcpkg
+RUN arrow/ci/scripts/install_vcpkg.sh ${VCPKG_ROOT} ${vcpkg} && \
  if [ "${manylinux}" == "2010" ]; then \
  arrow/ci/scripts/install_glibc.sh ${glibc} /opt/glibc-${glibc} && \
- patchelf --set-interpreter /opt/glibc-2.18/lib/ld-linux-x86-64.so.2 /opt/vcpkg/vcpkg && \
- patchelf --set-rpath /opt/glibc-2.18/lib:/usr/lib64 /opt/vcpkg/vcpkg; \
+ patchelf --set-interpreter /opt/glibc-2.18/lib/ld-linux-x86-64.so.2 ${VCPKG_ROOT}/vcpkg && \
+ patchelf --set-rpath /opt/glibc-2.18/lib:/usr/lib64 ${VCPKG_ROOT}/vcpkg; \
  fi
-ENV PATH="/opt/vcpkg:${PATH}"
+ENV PATH="${PATH}:${VCPKG_ROOT}"
 
 ARG build_type=release
 ENV CMAKE_BUILD_TYPE=${build_type} \
  VCPKG_FORCE_SYSTEM_BINARIES=1 \
  VCPKG_OVERLAY_TRIPLETS=/arrow/ci/vcpkg \
  VCPKG_DEFAULT_TRIPLET=${arch_short}-linux-static-${build_type} \
- VCPKG_FEATURE_FLAGS=-manifests
-
-# Need to install the boost-build prior installing the boost packages, otherwise
-# vcpkg will raise an error.
-# TODO(kszucs): factor out the package enumeration to a text file and reuse it
-# from the windows image and potentially in a future macos wheel build
-RUN vcpkg install --clean-after-build \
- abseil \
- aws-sdk-cpp[config,cognito-identity,core,identity-management,s3,sts,transfer] \
- boost-filesystem \
- brotli \
- bzip2 \
- c-ares \
- curl \
- flatbuffers \
- gflags \
- glog \
- google-cloud-cpp[core,storage] \
- grpc \
- lz4 \
- openssl \
- orc \
- protobuf \
- rapidjson \
- re2 \
- snappy \
- thrift \
- utf8proc \
- zlib \
- zstd
+ VCPKG_FEATURE_FLAGS="manifests"
+COPY ci/vcpkg/vcpkg.json arrow/ci/vcpkg/
+# cannot use the S3 feature here because while aws-sdk-cpp=1.9.160 contains
+# ssl related fixies as well as we can patch the vcpkg portfile to support
+# arm machines it hits ARROW-15141 where we would need to fall back to 1.8.186
+# but we cannot patch those portfiles since vcpkg-tool handles the checkout of
+# previous versions => use bundled S3 build
+RUN vcpkg install \
+ --clean-after-build \
+ --x-install-root=${VCPKG_ROOT}/installed \
+ --x-manifest-root=/arrow/ci/vcpkg \
+ --x-feature=flight \
+ --x-feature=gcs \
+ --x-feature=json \
+ --x-feature=parquet
 
 ARG python=3.8
 ENV PYTHON_VERSION=${python}

ci/docker/python-wheel-windows-vs2017.dockerfile

@@ -36,8 +36,9 @@ COPY ci/vcpkg/*.patch \
  ci/vcpkg/*windows*.cmake \
  arrow/ci/vcpkg/
 COPY ci/scripts/install_vcpkg.sh arrow/ci/scripts/
+ENV VCPKG_ROOT=C:\\vcpkg
 RUN bash arrow/ci/scripts/install_vcpkg.sh /c/vcpkg %vcpkg% && \
- setx PATH "%PATH%;C:\vcpkg"
+ setx PATH "%PATH%;%VCPKG_ROOT%"
 
 # Configure vcpkg and install dependencies
 # NOTE: use windows batch environment notation for build arguments in RUN
@@ -48,34 +49,21 @@ ARG build_type=release
 ENV CMAKE_BUILD_TYPE=${build_type} \
  VCPKG_OVERLAY_TRIPLETS=C:\\arrow\\ci\\vcpkg \
  VCPKG_DEFAULT_TRIPLET=amd64-windows-static-md-${build_type} \
- VCPKG_FEATURE_FLAGS=-manifests
-
-RUN vcpkg install --clean-after-build \
- abseil \
- aws-sdk-cpp[config,cognito-identity,core,identity-management,s3,sts,transfer] \
- boost-filesystem \
- boost-multiprecision \
- boost-system \
- brotli \
- bzip2 \
- c-ares \
- curl \
- flatbuffers \
- gflags \
- glog \
- google-cloud-cpp[core,storage] \
- grpc \
- lz4 \
- openssl \
- orc \
- protobuf \
- rapidjson \
- re2 \
- snappy \
- thrift \
- utf8proc \
- zlib \
- zstd
+ VCPKG_FEATURE_FLAGS="manifests"
+COPY ci/vcpkg/vcpkg.json arrow/ci/vcpkg/
+# cannot use the S3 feature here because while aws-sdk-cpp=1.9.160 contains
+# ssl related fixies as well as we can patch the vcpkg portfile to support
+# arm machines it hits ARROW-15141 where we would need to fall back to 1.8.186
+# but we cannot patch those portfiles since vcpkg-tool handles the checkout of
+# previous versions => use bundled S3 build
+RUN vcpkg install \
+ --clean-after-build \
+ --x-install-root=%VCPKG_ROOT%\installed \
+ --x-manifest-root=arrow/ci/vcpkg \
+ --x-feature=flight \
+ --x-feature=gcs \
+ --x-feature=json \
+ --x-feature=parquet
 
 # Remove previous installations of python from the base image
 # NOTE: a more recent base image (tried with 2.12.1) comes with python 3.9.7
@@ -87,20 +75,20 @@ RUN wmic product where "name like 'python%%'" call uninstall /nointeractive && \
 
 # Define the full version number otherwise choco falls back to patch number 0 (3.7 => 3.7.0)
 ARG python=3.8
-RUN (if "%python%"=="3.7" setx PYTHON_VERSION 3.7.12) & \
- (if "%python%"=="3.8" setx PYTHON_VERSION 3.8.11) & \
- (if "%python%"=="3.9" setx PYTHON_VERSION 3.9.9) & \
- (if "%python%"=="3.10" setx PYTHON_VERSION 3.10.1)
+RUN (if "%python%"=="3.7" setx PYTHON_VERSION "3.7.9" && setx PATH "%PATH%;C:\Python37;C:\Python37\Scripts") & \
+ (if "%python%"=="3.8" setx PYTHON_VERSION "3.8.10" && setx PATH "%PATH%;C:\Python38;C:\Python38\Scripts") & \
+ (if "%python%"=="3.9" setx PYTHON_VERSION "3.9.7" && setx PATH "%PATH%;C:\Python39;C:\Python39\Scripts") & \
+ (if "%python%"=="3.10" setx PYTHON_VERSION "3.10.2" && setx PATH "%PATH%;C:\Python310;C:\Python310\Scripts")
 RUN choco install -r -y --no-progress python --version=%PYTHON_VERSION%
-RUN pip install -U pip
+RUN python -m pip install -U pip setuptools
 
 COPY python/requirements-wheel-build.txt arrow/python/
-RUN pip install -r arrow/python/requirements-wheel-build.txt
+RUN python -m pip install -r arrow/python/requirements-wheel-build.txt
 
-ENV CLCACHE_DIR="C:\clcache"
-ENV CLCACHE_COMPRESS=1
-ENV CLCACHE_COMPRESSLEVEL=6
-RUN pip install git+https://github.com/Nuitka/clcache.git
+# ENV CLCACHE_DIR="C:\clcache"
+# ENV CLCACHE_COMPRESS=1
+# ENV CLCACHE_COMPRESSLEVEL=6
+# RUN pip install git+https://github.com/Nuitka/clcache.git
 
 # For debugging purposes
 # RUN wget --no-check-certificate https://github.com/lucasg/Dependencies/releases/download/v1.10/Dependencies_x64_Release.zip

ci/scripts/java_jni_manylinux_build.sh

@@ -55,9 +55,6 @@ export ARROW_TEST_DATA="${arrow_dir}/testing/data"
 export PARQUET_TEST_DATA="${arrow_dir}/cpp/submodules/parquet-testing/data"
 export AWS_EC2_METADATA_DISABLED=TRUE
 
-# NOTE(kszucs): workaround for ARROW-15403 along with the ORC_* cmake variables
-vcpkg remove orc
-
 mkdir -p "${build_dir}"
 pushd "${build_dir}"
 

ci/scripts/python_wheel_macos_build.sh

@@ -87,16 +87,11 @@ echo "=== (${PYTHON_VERSION}) Building Arrow C++ libraries ==="
 : ${VCPKG_FEATURE_FLAGS:=-manifests}
 : ${VCPKG_TARGET_TRIPLET:=${VCPKG_DEFAULT_TRIPLET:-x64-osx-static-${CMAKE_BUILD_TYPE}}}
 
-# NOTE(kszucs): workaround for ARROW-15403 along with the ORC_* cmake variables
-vcpkg remove orc
-
 mkdir -p ${build_dir}/build
 pushd ${build_dir}/build
 
 cmake \
  -DARROW_BUILD_SHARED=ON \
- -DCMAKE_APPLE_SILICON_PROCESSOR=arm64 \
- -DCMAKE_OSX_ARCHITECTURES=${CMAKE_OSX_ARCHITECTURES} \
  -DARROW_BUILD_STATIC=OFF \
  -DARROW_BUILD_TESTS=OFF \
  -DARROW_DATASET=${ARROW_DATASET} \
@@ -124,13 +119,16 @@ cmake \
  -DARROW_WITH_SNAPPY=${ARROW_WITH_SNAPPY} \
  -DARROW_WITH_ZLIB=${ARROW_WITH_ZLIB} \
  -DARROW_WITH_ZSTD=${ARROW_WITH_ZSTD} \
+ -DAWSSDK_SOURCE=BUNDLED \
+ -DCMAKE_APPLE_SILICON_PROCESSOR=arm64 \
  -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} \
  -DCMAKE_INSTALL_LIBDIR=lib \
  -DCMAKE_INSTALL_PREFIX=${build_dir}/install \
+ -DCMAKE_OSX_ARCHITECTURES=${CMAKE_OSX_ARCHITECTURES} \
  -DCMAKE_UNITY_BUILD=${CMAKE_UNITY_BUILD} \
  -DOPENSSL_USE_STATIC_LIBS=ON \
- -DORC_SOURCE=BUNDLED \
  -DORC_PROTOBUF_EXECUTABLE=${VCPKG_ROOT}/installed/${VCPKG_TARGET_TRIPLET}/tools/protobuf/protoc \
+ -DORC_SOURCE=BUNDLED \
  -DVCPKG_MANIFEST_MODE=OFF \
  -DVCPKG_TARGET_TRIPLET=${VCPKG_TARGET_TRIPLET} \
  -G ${CMAKE_GENERATOR} \

ci/scripts/python_wheel_manylinux_build.sh

@@ -80,11 +80,9 @@ if [[ "$(uname -m)" == arm* ]] || [[ "$(uname -m)" == aarch* ]]; then
  export ARROW_EXTRA_CMAKE_FLAGS="-DARROW_JEMALLOC_LG_PAGE=16"
 fi
 
-# NOTE(kszucs): workaround for ARROW-15403 along with the ORC_* cmake variables
-vcpkg remove orc
-
 mkdir /tmp/arrow-build
 pushd /tmp/arrow-build
+
 cmake \
  -DARROW_BROTLI_USE_SHARED=OFF \
  -DARROW_BUILD_SHARED=ON \
@@ -115,13 +113,14 @@ cmake \
  -DARROW_WITH_SNAPPY=${ARROW_WITH_SNAPPY} \
  -DARROW_WITH_ZLIB=${ARROW_WITH_ZLIB} \
  -DARROW_WITH_ZSTD=${ARROW_WITH_ZSTD} \
+ -DAWSSDK_SOURCE=BUNDLED \
  -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} \
  -DCMAKE_INSTALL_LIBDIR=lib \
  -DCMAKE_INSTALL_PREFIX=/tmp/arrow-dist \
  -DCMAKE_UNITY_BUILD=${CMAKE_UNITY_BUILD} \
  -DOPENSSL_USE_STATIC_LIBS=ON \
- -DORC_SOURCE=BUNDLED \
  -DORC_PROTOBUF_EXECUTABLE=${VCPKG_ROOT}/installed/${VCPKG_TARGET_TRIPLET}/tools/protobuf/protoc \
+ -DORC_SOURCE=BUNDLED \
  -DVCPKG_MANIFEST_MODE=OFF \
  -DVCPKG_TARGET_TRIPLET=${VCPKG_TARGET_TRIPLET} \
  ${ARROW_EXTRA_CMAKE_FLAGS} \

ci/scripts/python_wheel_windows_build.bat

@@ -51,9 +51,6 @@ set VCPKG_ROOT=C:\vcpkg
 set VCPKG_FEATURE_FLAGS=-manifests
 set VCGPK_TARGET_TRIPLET=amd64-windows-static-md-%CMAKE_BUILD_TYPE%
 
-@rem NOTE(kszucs): workaround for ARROW-15403 along with the ORC_* cmake variables
-vcpkg remove orc
-
 mkdir C:\arrow-build
 pushd C:\arrow-build
 cmake ^
@@ -80,13 +77,12 @@ cmake ^
  -DARROW_WITH_SNAPPY=%ARROW_WITH_SNAPPY% ^
  -DARROW_WITH_ZLIB=%ARROW_WITH_ZLIB% ^
  -DARROW_WITH_ZSTD=%ARROW_WITH_ZSTD% ^
+ -DAWSSDK_SOURCE=BUNDLED ^
  -DCMAKE_BUILD_TYPE=%CMAKE_BUILD_TYPE% ^
  -DCMAKE_CXX_COMPILER=clcache ^
  -DCMAKE_INSTALL_PREFIX=C:\arrow-dist ^
  -DCMAKE_UNITY_BUILD=%CMAKE_UNITY_BUILD% ^
  -DMSVC_LINK_VERBOSE=ON ^
- -DORC_SOURCE=BUNDLED ^
- -DORC_PROTOBUF_EXECUTABLE=%VCPKG_ROOT%\installed\%VCGPK_TARGET_TRIPLET%\tools\protobuf\protoc.exe ^
  -DVCPKG_MANIFEST_MODE=OFF ^
  -DVCPKG_TARGET_TRIPLET=%VCGPK_TARGET_TRIPLET% ^
  -G "%CMAKE_GENERATOR%" ^

ci/scripts/python_wheel_windows_test.bat

@@ -41,15 +41,15 @@ pip install -r C:\arrow\python\requirements-wheel-test.txt || exit /B
 python -m pip install --no-index --find-links=C:\arrow\python\dist\ pyarrow || exit /B
 
 @REM Test that the modules are importable
-python -c "import pyarrow"
-python -c "import pyarrow._hdfs"
-python -c "import pyarrow._s3fs"
-python -c "import pyarrow.csv"
-python -c "import pyarrow.dataset"
-python -c "import pyarrow.flight"
-python -c "import pyarrow.fs"
-python -c "import pyarrow.json"
-python -c "import pyarrow.parquet"
+python -c "import pyarrow" || exit /B
+python -c "import pyarrow._hdfs" || exit /B
+python -c "import pyarrow._s3fs" || exit /B
+python -c "import pyarrow.csv" || exit /B
+python -c "import pyarrow.dataset" || exit /B
+python -c "import pyarrow.flight" || exit /B
+python -c "import pyarrow.fs" || exit /B
+python -c "import pyarrow.json" || exit /B
+python -c "import pyarrow.parquet" || exit /B
 
 @REM Execute unittest
 pytest -r s --pyargs pyarrow || exit /B

ci/vcpkg/ports.patch

@@ -164,20 +164,20 @@ index 00000000..4566cc66
 +- " set(LibCrypto_SHARED_LIBRARY ${OPENSSL_CRYPTO_LIBRARY} CACHE INTERNAL \"The OpenSSL crypto shared library\")\n"
 +- "endif()\n"
 + )
-+
++
 + if (NOT SIMPLE_INSTALL)
 diff --git a/ports/aws-sdk-cpp/portfile.cmake b/ports/aws-sdk-cpp/portfile.cmake
 index 2d6bba4d..0ac47887 100644
 --- a/ports/aws-sdk-cpp/portfile.cmake
 +++ b/ports/aws-sdk-cpp/portfile.cmake
 @@ -8,6 +8,7 @@ vcpkg_from_github(
- PATCHES
- patch-relocatable-rpath.patch
- fix-aws-root.patch
-+ fix-find-crypto.patch
- )
-
- string(COMPARE EQUAL "${VCPKG_CRT_LINKAGE}" "dynamic" FORCE_SHARED_CRT)
+ PATCHES
+ patch-relocatable-rpath.patch
+ fix-aws-root.patch
++ fix-find-crypto.patch
+ )
+
+ string(COMPARE EQUAL "${VCPKG_CRT_LINKAGE}" "dynamic" FORCE_SHARED_CRT)
 diff --git a/ports/aws-sdk-cpp/vcpkg.json b/ports/aws-sdk-cpp/vcpkg.json
 index 3836e2b6..cd88ef07 100644
 --- a/ports/aws-sdk-cpp/vcpkg.json
@@ -216,7 +216,7 @@ index 45b8c706..b409d8a7 100644
  HEAD_REF master
 + PATCHES "snappy-disable-bmi.patch"
  )
-
+
  vcpkg_cmake_configure(
 diff --git a/ports/snappy/snappy-disable-bmi.patch b/ports/snappy/snappy-disable-bmi.patch
 new file mode 100644
@@ -241,5 +241,5 @@ index 00000000..a57ce0c2
 + return v & ~(mask << (8 * n));
 +-#endif
 + }
-+
++
 + static inline bool LeftShiftOverflows(uint8_t value, uint32_t shift) {

ci/docker/java-jni-manylinux-201x.dockerfile

@@ -19,19 +19,16 @@ ARG base
 FROM ${base}
 
 # Install the libaries required by the Gandiva to run
-RUN vcpkg install --clean-after-build \
- boost-algorithm \
- boost-crc \
- boost-date-time \
- boost-format \
- boost-locale \
- boost-multiprecision \
- boost-predef \
- boost-regex \
- boost-system \
- boost-variant \
- # Use enable rtti to avoid link problems in Gandiva
- llvm[clang,default-options,default-targets,lld,tools,enable-rtti]
+# Use enable llvm[enable-rtti] in the vcpkg.json to avoid link problems in Gandiva
+RUN vcpkg install \
+ --clean-after-build \
+ --x-install-root=${VCPKG_ROOT}/installed \
+ --x-manifest-root=/arrow/ci/vcpkg \
+ --x-feature=flight \
+ --x-feature=gcs \
+ --x-feature=json \
+ --x-feature=parquet \
+ --x-feature=gandiva
 
 # Install Java
 ARG java=1.8.0

ci/docker/python-wheel-manylinux-201x.dockerfile

@@ -53,49 +53,35 @@ COPY ci/vcpkg/*.patch \
 COPY ci/scripts/install_vcpkg.sh \
  ci/scripts/install_glibc.sh \
  arrow/ci/scripts/
-RUN arrow/ci/scripts/install_vcpkg.sh /opt/vcpkg ${vcpkg} && \
+ENV VCPKG_ROOT=/opt/vcpkg
+RUN arrow/ci/scripts/install_vcpkg.sh ${VCPKG_ROOT} ${vcpkg} && \
  if [ "${manylinux}" == "2010" ]; then \
  arrow/ci/scripts/install_glibc.sh ${glibc} /opt/glibc-${glibc} && \
- patchelf --set-interpreter /opt/glibc-2.18/lib/ld-linux-x86-64.so.2 /opt/vcpkg/vcpkg && \
- patchelf --set-rpath /opt/glibc-2.18/lib:/usr/lib64 /opt/vcpkg/vcpkg; \
+ patchelf --set-interpreter /opt/glibc-2.18/lib/ld-linux-x86-64.so.2 ${VCPKG_ROOT}/vcpkg && \
+ patchelf --set-rpath /opt/glibc-2.18/lib:/usr/lib64 ${VCPKG_ROOT}/vcpkg; \
  fi
-ENV PATH="/opt/vcpkg:${PATH}"
+ENV PATH="${PATH}:${VCPKG_ROOT}"
 
 ARG build_type=release
 ENV CMAKE_BUILD_TYPE=${build_type} \
  VCPKG_FORCE_SYSTEM_BINARIES=1 \
  VCPKG_OVERLAY_TRIPLETS=/arrow/ci/vcpkg \
  VCPKG_DEFAULT_TRIPLET=${arch_short}-linux-static-${build_type} \
- VCPKG_FEATURE_FLAGS=-manifests
-
-# Need to install the boost-build prior installing the boost packages, otherwise
-# vcpkg will raise an error.
-# TODO(kszucs): factor out the package enumeration to a text file and reuse it
-# from the windows image and potentially in a future macos wheel build
-RUN vcpkg install --clean-after-build \
- abseil \
- aws-sdk-cpp[config,cognito-identity,core,identity-management,s3,sts,transfer] \
- boost-filesystem \
- brotli \
- bzip2 \
- c-ares \
- curl \
- flatbuffers \
- gflags \
- glog \
- google-cloud-cpp[core,storage] \
- grpc \
- lz4 \
- openssl \
- orc \
- protobuf \
- rapidjson \
- re2 \
- snappy \
- thrift \
- utf8proc \
- zlib \
- zstd
+ VCPKG_FEATURE_FLAGS="manifests"
+COPY ci/vcpkg/vcpkg.json arrow/ci/vcpkg/
+# cannot use the S3 feature here because while aws-sdk-cpp=1.9.160 contains
+# ssl related fixies as well as we can patch the vcpkg portfile to support
+# arm machines it hits ARROW-15141 where we would need to fall back to 1.8.186
+# but we cannot patch those portfiles since vcpkg-tool handles the checkout of
+# previous versions => use bundled S3 build
+RUN vcpkg install \
+ --clean-after-build \
+ --x-install-root=${VCPKG_ROOT}/installed \
+ --x-manifest-root=/arrow/ci/vcpkg \
+ --x-feature=flight \
+ --x-feature=gcs \
+ --x-feature=json \
+ --x-feature=parquet
 
 ARG python=3.8
 ENV PYTHON_VERSION=${python}

ci/docker/python-wheel-windows-vs2017.dockerfile

@@ -36,8 +36,9 @@ COPY ci/vcpkg/*.patch \
  ci/vcpkg/*windows*.cmake \
  arrow/ci/vcpkg/
 COPY ci/scripts/install_vcpkg.sh arrow/ci/scripts/
+ENV VCPKG_ROOT=C:\\vcpkg
 RUN bash arrow/ci/scripts/install_vcpkg.sh /c/vcpkg %vcpkg% && \
- setx PATH "%PATH%;C:\vcpkg"
+ setx PATH "%PATH%;%VCPKG_ROOT%"
 
 # Configure vcpkg and install dependencies
 # NOTE: use windows batch environment notation for build arguments in RUN
@@ -48,34 +49,21 @@ ARG build_type=release
 ENV CMAKE_BUILD_TYPE=${build_type} \
  VCPKG_OVERLAY_TRIPLETS=C:\\arrow\\ci\\vcpkg \
  VCPKG_DEFAULT_TRIPLET=amd64-windows-static-md-${build_type} \
- VCPKG_FEATURE_FLAGS=-manifests
-
-RUN vcpkg install --clean-after-build \
- abseil \
- aws-sdk-cpp[config,cognito-identity,core,identity-management,s3,sts,transfer] \
- boost-filesystem \
- boost-multiprecision \
- boost-system \
- brotli \
- bzip2 \
- c-ares \
- curl \
- flatbuffers \
- gflags \
- glog \
- google-cloud-cpp[core,storage] \
- grpc \
- lz4 \
- openssl \
- orc \
- protobuf \
- rapidjson \
- re2 \
- snappy \
- thrift \
- utf8proc \
- zlib \
- zstd
+ VCPKG_FEATURE_FLAGS="manifests"
+COPY ci/vcpkg/vcpkg.json arrow/ci/vcpkg/
+# cannot use the S3 feature here because while aws-sdk-cpp=1.9.160 contains
+# ssl related fixies as well as we can patch the vcpkg portfile to support
+# arm machines it hits ARROW-15141 where we would need to fall back to 1.8.186
+# but we cannot patch those portfiles since vcpkg-tool handles the checkout of
+# previous versions => use bundled S3 build
+RUN vcpkg install \
+ --clean-after-build \
+ --x-install-root=%VCPKG_ROOT%\installed \
+ --x-manifest-root=arrow/ci/vcpkg \
+ --x-feature=flight \
+ --x-feature=gcs \
+ --x-feature=json \
+ --x-feature=parquet
 
 # Remove previous installations of python from the base image
 # NOTE: a more recent base image (tried with 2.12.1) comes with python 3.9.7
@@ -87,20 +75,20 @@ RUN wmic product where "name like 'python%%'" call uninstall /nointeractive && \
 
 # Define the full version number otherwise choco falls back to patch number 0 (3.7 => 3.7.0)
 ARG python=3.8
-RUN (if "%python%"=="3.7" setx PYTHON_VERSION 3.7.12) & \
- (if "%python%"=="3.8" setx PYTHON_VERSION 3.8.11) & \
- (if "%python%"=="3.9" setx PYTHON_VERSION 3.9.9) & \
- (if "%python%"=="3.10" setx PYTHON_VERSION 3.10.1)
+RUN (if "%python%"=="3.7" setx PYTHON_VERSION "3.7.9" && setx PATH "%PATH%;C:\Python37;C:\Python37\Scripts") & \
+ (if "%python%"=="3.8" setx PYTHON_VERSION "3.8.10" && setx PATH "%PATH%;C:\Python38;C:\Python38\Scripts") & \
+ (if "%python%"=="3.9" setx PYTHON_VERSION "3.9.7" && setx PATH "%PATH%;C:\Python39;C:\Python39\Scripts") & \
+ (if "%python%"=="3.10" setx PYTHON_VERSION "3.10.2" && setx PATH "%PATH%;C:\Python310;C:\Python310\Scripts")
 RUN choco install -r -y --no-progress python --version=%PYTHON_VERSION%
-RUN pip install -U pip
+RUN python -m pip install -U pip setuptools
 
 COPY python/requirements-wheel-build.txt arrow/python/
-RUN pip install -r arrow/python/requirements-wheel-build.txt
+RUN python -m pip install -r arrow/python/requirements-wheel-build.txt
 
-ENV CLCACHE_DIR="C:\clcache"
-ENV CLCACHE_COMPRESS=1
-ENV CLCACHE_COMPRESSLEVEL=6
-RUN pip install git+https://github.com/Nuitka/clcache.git
+# ENV CLCACHE_DIR="C:\clcache"
+# ENV CLCACHE_COMPRESS=1
+# ENV CLCACHE_COMPRESSLEVEL=6
+# RUN pip install git+https://github.com/Nuitka/clcache.git
 
 # For debugging purposes
 # RUN wget --no-check-certificate https://github.com/lucasg/Dependencies/releases/download/v1.10/Dependencies_x64_Release.zip

ci/scripts/java_jni_manylinux_build.sh

@@ -55,9 +55,6 @@ export ARROW_TEST_DATA="${arrow_dir}/testing/data"
 export PARQUET_TEST_DATA="${arrow_dir}/cpp/submodules/parquet-testing/data"
 export AWS_EC2_METADATA_DISABLED=TRUE
 
-# NOTE(kszucs): workaround for ARROW-15403 along with the ORC_* cmake variables
-vcpkg remove orc
-
 mkdir -p "${build_dir}"
 pushd "${build_dir}"
 

ci/scripts/python_wheel_macos_build.sh

@@ -87,16 +87,11 @@ echo "=== (${PYTHON_VERSION}) Building Arrow C++ libraries ==="
 : ${VCPKG_FEATURE_FLAGS:=-manifests}
 : ${VCPKG_TARGET_TRIPLET:=${VCPKG_DEFAULT_TRIPLET:-x64-osx-static-${CMAKE_BUILD_TYPE}}}
 
-# NOTE(kszucs): workaround for ARROW-15403 along with the ORC_* cmake variables
-vcpkg remove orc
-
 mkdir -p ${build_dir}/build
 pushd ${build_dir}/build
 
 cmake \
  -DARROW_BUILD_SHARED=ON \
- -DCMAKE_APPLE_SILICON_PROCESSOR=arm64 \
- -DCMAKE_OSX_ARCHITECTURES=${CMAKE_OSX_ARCHITECTURES} \
  -DARROW_BUILD_STATIC=OFF \
  -DARROW_BUILD_TESTS=OFF \
  -DARROW_DATASET=${ARROW_DATASET} \
@@ -124,13 +119,16 @@ cmake \
  -DARROW_WITH_SNAPPY=${ARROW_WITH_SNAPPY} \
  -DARROW_WITH_ZLIB=${ARROW_WITH_ZLIB} \
  -DARROW_WITH_ZSTD=${ARROW_WITH_ZSTD} \
+ -DAWSSDK_SOURCE=BUNDLED \
+ -DCMAKE_APPLE_SILICON_PROCESSOR=arm64 \
  -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} \
  -DCMAKE_INSTALL_LIBDIR=lib \
  -DCMAKE_INSTALL_PREFIX=${build_dir}/install \
+ -DCMAKE_OSX_ARCHITECTURES=${CMAKE_OSX_ARCHITECTURES} \
  -DCMAKE_UNITY_BUILD=${CMAKE_UNITY_BUILD} \
  -DOPENSSL_USE_STATIC_LIBS=ON \
- -DORC_SOURCE=BUNDLED \
  -DORC_PROTOBUF_EXECUTABLE=${VCPKG_ROOT}/installed/${VCPKG_TARGET_TRIPLET}/tools/protobuf/protoc \
+ -DORC_SOURCE=BUNDLED \
  -DVCPKG_MANIFEST_MODE=OFF \
  -DVCPKG_TARGET_TRIPLET=${VCPKG_TARGET_TRIPLET} \
  -G ${CMAKE_GENERATOR} \

ci/scripts/python_wheel_manylinux_build.sh

@@ -80,11 +80,9 @@ if [[ "$(uname -m)" == arm* ]] || [[ "$(uname -m)" == aarch* ]]; then
  export ARROW_EXTRA_CMAKE_FLAGS="-DARROW_JEMALLOC_LG_PAGE=16"
 fi
 
-# NOTE(kszucs): workaround for ARROW-15403 along with the ORC_* cmake variables
-vcpkg remove orc
-
 mkdir /tmp/arrow-build
 pushd /tmp/arrow-build
+
 cmake \
  -DARROW_BROTLI_USE_SHARED=OFF \
  -DARROW_BUILD_SHARED=ON \
@@ -115,13 +113,14 @@ cmake \
  -DARROW_WITH_SNAPPY=${ARROW_WITH_SNAPPY} \
  -DARROW_WITH_ZLIB=${ARROW_WITH_ZLIB} \
  -DARROW_WITH_ZSTD=${ARROW_WITH_ZSTD} \
+ -DAWSSDK_SOURCE=BUNDLED \
  -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} \
  -DCMAKE_INSTALL_LIBDIR=lib \
  -DCMAKE_INSTALL_PREFIX=/tmp/arrow-dist \
  -DCMAKE_UNITY_BUILD=${CMAKE_UNITY_BUILD} \
  -DOPENSSL_USE_STATIC_LIBS=ON \
- -DORC_SOURCE=BUNDLED \
  -DORC_PROTOBUF_EXECUTABLE=${VCPKG_ROOT}/installed/${VCPKG_TARGET_TRIPLET}/tools/protobuf/protoc \
+ -DORC_SOURCE=BUNDLED \
  -DVCPKG_MANIFEST_MODE=OFF \
  -DVCPKG_TARGET_TRIPLET=${VCPKG_TARGET_TRIPLET} \
  ${ARROW_EXTRA_CMAKE_FLAGS} \

ci/scripts/python_wheel_windows_build.bat

@@ -51,9 +51,6 @@ set VCPKG_ROOT=C:\vcpkg
 set VCPKG_FEATURE_FLAGS=-manifests
 set VCGPK_TARGET_TRIPLET=amd64-windows-static-md-%CMAKE_BUILD_TYPE%
 
-@rem NOTE(kszucs): workaround for ARROW-15403 along with the ORC_* cmake variables
-vcpkg remove orc
-
 mkdir C:\arrow-build
 pushd C:\arrow-build
 cmake ^
@@ -80,13 +77,12 @@ cmake ^
  -DARROW_WITH_SNAPPY=%ARROW_WITH_SNAPPY% ^
  -DARROW_WITH_ZLIB=%ARROW_WITH_ZLIB% ^
  -DARROW_WITH_ZSTD=%ARROW_WITH_ZSTD% ^
+ -DAWSSDK_SOURCE=BUNDLED ^
  -DCMAKE_BUILD_TYPE=%CMAKE_BUILD_TYPE% ^
  -DCMAKE_CXX_COMPILER=clcache ^
  -DCMAKE_INSTALL_PREFIX=C:\arrow-dist ^
  -DCMAKE_UNITY_BUILD=%CMAKE_UNITY_BUILD% ^
  -DMSVC_LINK_VERBOSE=ON ^
- -DORC_SOURCE=BUNDLED ^
- -DORC_PROTOBUF_EXECUTABLE=%VCPKG_ROOT%\installed\%VCGPK_TARGET_TRIPLET%\tools\protobuf\protoc.exe ^
  -DVCPKG_MANIFEST_MODE=OFF ^
  -DVCPKG_TARGET_TRIPLET=%VCGPK_TARGET_TRIPLET% ^
  -G "%CMAKE_GENERATOR%" ^

ci/scripts/python_wheel_windows_test.bat

@@ -41,15 +41,15 @@ pip install -r C:\arrow\python\requirements-wheel-test.txt || exit /B
 python -m pip install --no-index --find-links=C:\arrow\python\dist\ pyarrow || exit /B
 
 @REM Test that the modules are importable
-python -c "import pyarrow"
-python -c "import pyarrow._hdfs"
-python -c "import pyarrow._s3fs"
-python -c "import pyarrow.csv"
-python -c "import pyarrow.dataset"
-python -c "import pyarrow.flight"
-python -c "import pyarrow.fs"
-python -c "import pyarrow.json"
-python -c "import pyarrow.parquet"
+python -c "import pyarrow" || exit /B
+python -c "import pyarrow._hdfs" || exit /B
+python -c "import pyarrow._s3fs" || exit /B
+python -c "import pyarrow.csv" || exit /B
+python -c "import pyarrow.dataset" || exit /B
+python -c "import pyarrow.flight" || exit /B
+python -c "import pyarrow.fs" || exit /B
+python -c "import pyarrow.json" || exit /B
+python -c "import pyarrow.parquet" || exit /B
 
 @REM Execute unittest
 pytest -r s --pyargs pyarrow || exit /B

ci/vcpkg/ports.patch

@@ -164,20 +164,20 @@ index 00000000..4566cc66
 +- " set(LibCrypto_SHARED_LIBRARY ${OPENSSL_CRYPTO_LIBRARY} CACHE INTERNAL \"The OpenSSL crypto shared library\")\n"
 +- "endif()\n"
 + )
-+
++
 + if (NOT SIMPLE_INSTALL)
 diff --git a/ports/aws-sdk-cpp/portfile.cmake b/ports/aws-sdk-cpp/portfile.cmake
 index 2d6bba4d..0ac47887 100644
 --- a/ports/aws-sdk-cpp/portfile.cmake
 +++ b/ports/aws-sdk-cpp/portfile.cmake
 @@ -8,6 +8,7 @@ vcpkg_from_github(
- PATCHES
- patch-relocatable-rpath.patch
- fix-aws-root.patch
-+ fix-find-crypto.patch
- )
-
- string(COMPARE EQUAL "${VCPKG_CRT_LINKAGE}" "dynamic" FORCE_SHARED_CRT)
+ PATCHES
+ patch-relocatable-rpath.patch
+ fix-aws-root.patch
++ fix-find-crypto.patch
+ )
+
+ string(COMPARE EQUAL "${VCPKG_CRT_LINKAGE}" "dynamic" FORCE_SHARED_CRT)
 diff --git a/ports/aws-sdk-cpp/vcpkg.json b/ports/aws-sdk-cpp/vcpkg.json
 index 3836e2b6..cd88ef07 100644
 --- a/ports/aws-sdk-cpp/vcpkg.json
@@ -216,7 +216,7 @@ index 45b8c706..b409d8a7 100644
  HEAD_REF master
 + PATCHES "snappy-disable-bmi.patch"
  )
-
+
  vcpkg_cmake_configure(
 diff --git a/ports/snappy/snappy-disable-bmi.patch b/ports/snappy/snappy-disable-bmi.patch
 new file mode 100644
@@ -241,5 +241,5 @@ index 00000000..a57ce0c2
 + return v & ~(mask << (8 * n));
 +-#endif
 + }
-+
++
 + static inline bool LeftShiftOverflows(uint8_t value, uint32_t shift) {

ci/docker/java-jni-manylinux-201x.dockerfile

@@ -19,19 +19,16 @@ ARG base
 FROM ${base}
 
 # Install the libaries required by the Gandiva to run
-RUN vcpkg install --clean-after-build \
- boost-algorithm \
- boost-crc \
- boost-date-time \
- boost-format \
- boost-locale \
- boost-multiprecision \
- boost-predef \
- boost-regex \
- boost-system \
- boost-variant \
- # Use enable rtti to avoid link problems in Gandiva
- llvm[clang,default-options,default-targets,lld,tools,enable-rtti]
+# Use enable llvm[enable-rtti] in the vcpkg.json to avoid link problems in Gandiva
+RUN vcpkg install \
+ --clean-after-build \
+ --x-install-root=${VCPKG_ROOT}/installed \
+ --x-manifest-root=/arrow/ci/vcpkg \
+ --x-feature=flight \
+ --x-feature=gcs \
+ --x-feature=json \
+ --x-feature=parquet \
+ --x-feature=gandiva
 
 # Install Java
 ARG java=1.8.0

ci/docker/python-wheel-manylinux-201x.dockerfile

@@ -53,49 +53,35 @@ COPY ci/vcpkg/*.patch \
 COPY ci/scripts/install_vcpkg.sh \
  ci/scripts/install_glibc.sh \
  arrow/ci/scripts/
-RUN arrow/ci/scripts/install_vcpkg.sh /opt/vcpkg ${vcpkg} && \
+ENV VCPKG_ROOT=/opt/vcpkg
+RUN arrow/ci/scripts/install_vcpkg.sh ${VCPKG_ROOT} ${vcpkg} && \
  if [ "${manylinux}" == "2010" ]; then \
  arrow/ci/scripts/install_glibc.sh ${glibc} /opt/glibc-${glibc} && \
- patchelf --set-interpreter /opt/glibc-2.18/lib/ld-linux-x86-64.so.2 /opt/vcpkg/vcpkg && \
- patchelf --set-rpath /opt/glibc-2.18/lib:/usr/lib64 /opt/vcpkg/vcpkg; \
+ patchelf --set-interpreter /opt/glibc-2.18/lib/ld-linux-x86-64.so.2 ${VCPKG_ROOT}/vcpkg && \
+ patchelf --set-rpath /opt/glibc-2.18/lib:/usr/lib64 ${VCPKG_ROOT}/vcpkg; \
  fi
-ENV PATH="/opt/vcpkg:${PATH}"
+ENV PATH="${PATH}:${VCPKG_ROOT}"
 
 ARG build_type=release
 ENV CMAKE_BUILD_TYPE=${build_type} \
  VCPKG_FORCE_SYSTEM_BINARIES=1 \
  VCPKG_OVERLAY_TRIPLETS=/arrow/ci/vcpkg \
  VCPKG_DEFAULT_TRIPLET=${arch_short}-linux-static-${build_type} \
- VCPKG_FEATURE_FLAGS=-manifests
-
-# Need to install the boost-build prior installing the boost packages, otherwise
-# vcpkg will raise an error.
-# TODO(kszucs): factor out the package enumeration to a text file and reuse it
-# from the windows image and potentially in a future macos wheel build
-RUN vcpkg install --clean-after-build \
- abseil \
- aws-sdk-cpp[config,cognito-identity,core,identity-management,s3,sts,transfer] \
- boost-filesystem \
- brotli \
- bzip2 \
- c-ares \
- curl \
- flatbuffers \
- gflags \
- glog \
- google-cloud-cpp[core,storage] \
- grpc \
- lz4 \
- openssl \
- orc \
- protobuf \
- rapidjson \
- re2 \
- snappy \
- thrift \
- utf8proc \
- zlib \
- zstd
+ VCPKG_FEATURE_FLAGS="manifests"
+COPY ci/vcpkg/vcpkg.json arrow/ci/vcpkg/
+# cannot use the S3 feature here because while aws-sdk-cpp=1.9.160 contains
+# ssl related fixies as well as we can patch the vcpkg portfile to support
+# arm machines it hits ARROW-15141 where we would need to fall back to 1.8.186
+# but we cannot patch those portfiles since vcpkg-tool handles the checkout of
+# previous versions => use bundled S3 build
+RUN vcpkg install \
+ --clean-after-build \
+ --x-install-root=${VCPKG_ROOT}/installed \
+ --x-manifest-root=/arrow/ci/vcpkg \
+ --x-feature=flight \
+ --x-feature=gcs \
+ --x-feature=json \
+ --x-feature=parquet
 
 ARG python=3.8
 ENV PYTHON_VERSION=${python}

ci/docker/python-wheel-windows-vs2017.dockerfile

@@ -36,8 +36,9 @@ COPY ci/vcpkg/*.patch \
  ci/vcpkg/*windows*.cmake \
  arrow/ci/vcpkg/
 COPY ci/scripts/install_vcpkg.sh arrow/ci/scripts/
+ENV VCPKG_ROOT=C:\\vcpkg
 RUN bash arrow/ci/scripts/install_vcpkg.sh /c/vcpkg %vcpkg% && \
- setx PATH "%PATH%;C:\vcpkg"
+ setx PATH "%PATH%;%VCPKG_ROOT%"
 
 # Configure vcpkg and install dependencies
 # NOTE: use windows batch environment notation for build arguments in RUN
@@ -48,34 +49,21 @@ ARG build_type=release
 ENV CMAKE_BUILD_TYPE=${build_type} \
  VCPKG_OVERLAY_TRIPLETS=C:\\arrow\\ci\\vcpkg \
  VCPKG_DEFAULT_TRIPLET=amd64-windows-static-md-${build_type} \
- VCPKG_FEATURE_FLAGS=-manifests
-
-RUN vcpkg install --clean-after-build \
- abseil \
- aws-sdk-cpp[config,cognito-identity,core,identity-management,s3,sts,transfer] \
- boost-filesystem \
- boost-multiprecision \
- boost-system \
- brotli \
- bzip2 \
- c-ares \
- curl \
- flatbuffers \
- gflags \
- glog \
- google-cloud-cpp[core,storage] \
- grpc \
- lz4 \
- openssl \
- orc \
- protobuf \
- rapidjson \
- re2 \
- snappy \
- thrift \
- utf8proc \
- zlib \
- zstd
+ VCPKG_FEATURE_FLAGS="manifests"
+COPY ci/vcpkg/vcpkg.json arrow/ci/vcpkg/
+# cannot use the S3 feature here because while aws-sdk-cpp=1.9.160 contains
+# ssl related fixies as well as we can patch the vcpkg portfile to support
+# arm machines it hits ARROW-15141 where we would need to fall back to 1.8.186
+# but we cannot patch those portfiles since vcpkg-tool handles the checkout of
+# previous versions => use bundled S3 build
+RUN vcpkg install \
+ --clean-after-build \
+ --x-install-root=%VCPKG_ROOT%\installed \
+ --x-manifest-root=arrow/ci/vcpkg \
+ --x-feature=flight \
+ --x-feature=gcs \
+ --x-feature=json \
+ --x-feature=parquet
 
 # Remove previous installations of python from the base image
 # NOTE: a more recent base image (tried with 2.12.1) comes with python 3.9.7
@@ -87,20 +75,20 @@ RUN wmic product where "name like 'python%%'" call uninstall /nointeractive && \
 
 # Define the full version number otherwise choco falls back to patch number 0 (3.7 => 3.7.0)
 ARG python=3.8
-RUN (if "%python%"=="3.7" setx PYTHON_VERSION 3.7.12) & \
- (if "%python%"=="3.8" setx PYTHON_VERSION 3.8.11) & \
- (if "%python%"=="3.9" setx PYTHON_VERSION 3.9.9) & \
- (if "%python%"=="3.10" setx PYTHON_VERSION 3.10.1)
+RUN (if "%python%"=="3.7" setx PYTHON_VERSION "3.7.9" && setx PATH "%PATH%;C:\Python37;C:\Python37\Scripts") & \
+ (if "%python%"=="3.8" setx PYTHON_VERSION "3.8.10" && setx PATH "%PATH%;C:\Python38;C:\Python38\Scripts") & \
+ (if "%python%"=="3.9" setx PYTHON_VERSION "3.9.7" && setx PATH "%PATH%;C:\Python39;C:\Python39\Scripts") & \
+ (if "%python%"=="3.10" setx PYTHON_VERSION "3.10.2" && setx PATH "%PATH%;C:\Python310;C:\Python310\Scripts")
 RUN choco install -r -y --no-progress python --version=%PYTHON_VERSION%
-RUN pip install -U pip
+RUN python -m pip install -U pip setuptools
 
 COPY python/requirements-wheel-build.txt arrow/python/
-RUN pip install -r arrow/python/requirements-wheel-build.txt
+RUN python -m pip install -r arrow/python/requirements-wheel-build.txt
 
-ENV CLCACHE_DIR="C:\clcache"
-ENV CLCACHE_COMPRESS=1
-ENV CLCACHE_COMPRESSLEVEL=6
-RUN pip install git+https://github.com/Nuitka/clcache.git
+# ENV CLCACHE_DIR="C:\clcache"
+# ENV CLCACHE_COMPRESS=1
+# ENV CLCACHE_COMPRESSLEVEL=6
+# RUN pip install git+https://github.com/Nuitka/clcache.git
 
 # For debugging purposes
 # RUN wget --no-check-certificate https://github.com/lucasg/Dependencies/releases/download/v1.10/Dependencies_x64_Release.zip

ci/scripts/java_jni_manylinux_build.sh

@@ -55,9 +55,6 @@ export ARROW_TEST_DATA="${arrow_dir}/testing/data"
 export PARQUET_TEST_DATA="${arrow_dir}/cpp/submodules/parquet-testing/data"
 export AWS_EC2_METADATA_DISABLED=TRUE
 
-# NOTE(kszucs): workaround for ARROW-15403 along with the ORC_* cmake variables
-vcpkg remove orc
-
 mkdir -p "${build_dir}"
 pushd "${build_dir}"
 

ci/scripts/python_wheel_macos_build.sh

@@ -87,16 +87,11 @@ echo "=== (${PYTHON_VERSION}) Building Arrow C++ libraries ==="
 : ${VCPKG_FEATURE_FLAGS:=-manifests}
 : ${VCPKG_TARGET_TRIPLET:=${VCPKG_DEFAULT_TRIPLET:-x64-osx-static-${CMAKE_BUILD_TYPE}}}
 
-# NOTE(kszucs): workaround for ARROW-15403 along with the ORC_* cmake variables
-vcpkg remove orc
-
 mkdir -p ${build_dir}/build
 pushd ${build_dir}/build
 
 cmake \
  -DARROW_BUILD_SHARED=ON \
- -DCMAKE_APPLE_SILICON_PROCESSOR=arm64 \
- -DCMAKE_OSX_ARCHITECTURES=${CMAKE_OSX_ARCHITECTURES} \
  -DARROW_BUILD_STATIC=OFF \
  -DARROW_BUILD_TESTS=OFF \
  -DARROW_DATASET=${ARROW_DATASET} \
@@ -124,13 +119,16 @@ cmake \
  -DARROW_WITH_SNAPPY=${ARROW_WITH_SNAPPY} \
  -DARROW_WITH_ZLIB=${ARROW_WITH_ZLIB} \
  -DARROW_WITH_ZSTD=${ARROW_WITH_ZSTD} \
+ -DAWSSDK_SOURCE=BUNDLED \
+ -DCMAKE_APPLE_SILICON_PROCESSOR=arm64 \
  -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} \
  -DCMAKE_INSTALL_LIBDIR=lib \
  -DCMAKE_INSTALL_PREFIX=${build_dir}/install \
+ -DCMAKE_OSX_ARCHITECTURES=${CMAKE_OSX_ARCHITECTURES} \
  -DCMAKE_UNITY_BUILD=${CMAKE_UNITY_BUILD} \
  -DOPENSSL_USE_STATIC_LIBS=ON \
- -DORC_SOURCE=BUNDLED \
  -DORC_PROTOBUF_EXECUTABLE=${VCPKG_ROOT}/installed/${VCPKG_TARGET_TRIPLET}/tools/protobuf/protoc \
+ -DORC_SOURCE=BUNDLED \
  -DVCPKG_MANIFEST_MODE=OFF \
  -DVCPKG_TARGET_TRIPLET=${VCPKG_TARGET_TRIPLET} \
  -G ${CMAKE_GENERATOR} \

ci/scripts/python_wheel_manylinux_build.sh

@@ -80,11 +80,9 @@ if [[ "$(uname -m)" == arm* ]] || [[ "$(uname -m)" == aarch* ]]; then
  export ARROW_EXTRA_CMAKE_FLAGS="-DARROW_JEMALLOC_LG_PAGE=16"
 fi
 
-# NOTE(kszucs): workaround for ARROW-15403 along with the ORC_* cmake variables
-vcpkg remove orc
-
 mkdir /tmp/arrow-build
 pushd /tmp/arrow-build
+
 cmake \
  -DARROW_BROTLI_USE_SHARED=OFF \
  -DARROW_BUILD_SHARED=ON \
@@ -115,13 +113,14 @@ cmake \
  -DARROW_WITH_SNAPPY=${ARROW_WITH_SNAPPY} \
  -DARROW_WITH_ZLIB=${ARROW_WITH_ZLIB} \
  -DARROW_WITH_ZSTD=${ARROW_WITH_ZSTD} \
+ -DAWSSDK_SOURCE=BUNDLED \
  -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} \
  -DCMAKE_INSTALL_LIBDIR=lib \
  -DCMAKE_INSTALL_PREFIX=/tmp/arrow-dist \
  -DCMAKE_UNITY_BUILD=${CMAKE_UNITY_BUILD} \
  -DOPENSSL_USE_STATIC_LIBS=ON \
- -DORC_SOURCE=BUNDLED \
  -DORC_PROTOBUF_EXECUTABLE=${VCPKG_ROOT}/installed/${VCPKG_TARGET_TRIPLET}/tools/protobuf/protoc \
+ -DORC_SOURCE=BUNDLED \
  -DVCPKG_MANIFEST_MODE=OFF \
  -DVCPKG_TARGET_TRIPLET=${VCPKG_TARGET_TRIPLET} \
  ${ARROW_EXTRA_CMAKE_FLAGS} \

ci/scripts/python_wheel_windows_build.bat

@@ -51,9 +51,6 @@ set VCPKG_ROOT=C:\vcpkg
 set VCPKG_FEATURE_FLAGS=-manifests
 set VCGPK_TARGET_TRIPLET=amd64-windows-static-md-%CMAKE_BUILD_TYPE%
 
-@rem NOTE(kszucs): workaround for ARROW-15403 along with the ORC_* cmake variables
-vcpkg remove orc
-
 mkdir C:\arrow-build
 pushd C:\arrow-build
 cmake ^
@@ -80,13 +77,12 @@ cmake ^
  -DARROW_WITH_SNAPPY=%ARROW_WITH_SNAPPY% ^
  -DARROW_WITH_ZLIB=%ARROW_WITH_ZLIB% ^
  -DARROW_WITH_ZSTD=%ARROW_WITH_ZSTD% ^
+ -DAWSSDK_SOURCE=BUNDLED ^
  -DCMAKE_BUILD_TYPE=%CMAKE_BUILD_TYPE% ^
  -DCMAKE_CXX_COMPILER=clcache ^
  -DCMAKE_INSTALL_PREFIX=C:\arrow-dist ^
  -DCMAKE_UNITY_BUILD=%CMAKE_UNITY_BUILD% ^
  -DMSVC_LINK_VERBOSE=ON ^
- -DORC_SOURCE=BUNDLED ^
- -DORC_PROTOBUF_EXECUTABLE=%VCPKG_ROOT%\installed\%VCGPK_TARGET_TRIPLET%\tools\protobuf\protoc.exe ^
  -DVCPKG_MANIFEST_MODE=OFF ^
  -DVCPKG_TARGET_TRIPLET=%VCGPK_TARGET_TRIPLET% ^
  -G "%CMAKE_GENERATOR%" ^

ci/scripts/python_wheel_windows_test.bat

@@ -41,15 +41,15 @@ pip install -r C:\arrow\python\requirements-wheel-test.txt || exit /B
 python -m pip install --no-index --find-links=C:\arrow\python\dist\ pyarrow || exit /B
 
 @REM Test that the modules are importable
-python -c "import pyarrow"
-python -c "import pyarrow._hdfs"
-python -c "import pyarrow._s3fs"
-python -c "import pyarrow.csv"
-python -c "import pyarrow.dataset"
-python -c "import pyarrow.flight"
-python -c "import pyarrow.fs"
-python -c "import pyarrow.json"
-python -c "import pyarrow.parquet"
+python -c "import pyarrow" || exit /B
+python -c "import pyarrow._hdfs" || exit /B
+python -c "import pyarrow._s3fs" || exit /B
+python -c "import pyarrow.csv" || exit /B
+python -c "import pyarrow.dataset" || exit /B
+python -c "import pyarrow.flight" || exit /B
+python -c "import pyarrow.fs" || exit /B
+python -c "import pyarrow.json" || exit /B
+python -c "import pyarrow.parquet" || exit /B
 
 @REM Execute unittest
 pytest -r s --pyargs pyarrow || exit /B

ci/vcpkg/ports.patch

@@ -164,20 +164,20 @@ index 00000000..4566cc66
 +- " set(LibCrypto_SHARED_LIBRARY ${OPENSSL_CRYPTO_LIBRARY} CACHE INTERNAL \"The OpenSSL crypto shared library\")\n"
 +- "endif()\n"
 + )
-+
++
 + if (NOT SIMPLE_INSTALL)
 diff --git a/ports/aws-sdk-cpp/portfile.cmake b/ports/aws-sdk-cpp/portfile.cmake
 index 2d6bba4d..0ac47887 100644
 --- a/ports/aws-sdk-cpp/portfile.cmake
 +++ b/ports/aws-sdk-cpp/portfile.cmake
 @@ -8,6 +8,7 @@ vcpkg_from_github(
- PATCHES
- patch-relocatable-rpath.patch
- fix-aws-root.patch
-+ fix-find-crypto.patch
- )
-
- string(COMPARE EQUAL "${VCPKG_CRT_LINKAGE}" "dynamic" FORCE_SHARED_CRT)
+ PATCHES
+ patch-relocatable-rpath.patch
+ fix-aws-root.patch
++ fix-find-crypto.patch
+ )
+
+ string(COMPARE EQUAL "${VCPKG_CRT_LINKAGE}" "dynamic" FORCE_SHARED_CRT)
 diff --git a/ports/aws-sdk-cpp/vcpkg.json b/ports/aws-sdk-cpp/vcpkg.json
 index 3836e2b6..cd88ef07 100644
 --- a/ports/aws-sdk-cpp/vcpkg.json
@@ -216,7 +216,7 @@ index 45b8c706..b409d8a7 100644
  HEAD_REF master
 + PATCHES "snappy-disable-bmi.patch"
  )
-
+
  vcpkg_cmake_configure(
 diff --git a/ports/snappy/snappy-disable-bmi.patch b/ports/snappy/snappy-disable-bmi.patch
 new file mode 100644
@@ -241,5 +241,5 @@ index 00000000..a57ce0c2
 + return v & ~(mask << (8 * n));
 +-#endif
 + }
-+
++
 + static inline bool LeftShiftOverflows(uint8_t value, uint32_t shift) {

ci/docker/java-jni-manylinux-201x.dockerfile

@@ -19,19 +19,16 @@ ARG base
 FROM ${base}
 
 # Install the libaries required by the Gandiva to run
-RUN vcpkg install --clean-after-build \
- boost-algorithm \
- boost-crc \
- boost-date-time \
- boost-format \
- boost-locale \
- boost-multiprecision \
- boost-predef \
- boost-regex \
- boost-system \
- boost-variant \
- # Use enable rtti to avoid link problems in Gandiva
- llvm[clang,default-options,default-targets,lld,tools,enable-rtti]
+# Use enable llvm[enable-rtti] in the vcpkg.json to avoid link problems in Gandiva
+RUN vcpkg install \
+ --clean-after-build \
+ --x-install-root=${VCPKG_ROOT}/installed \
+ --x-manifest-root=/arrow/ci/vcpkg \
+ --x-feature=flight \
+ --x-feature=gcs \
+ --x-feature=json \
+ --x-feature=parquet \
+ --x-feature=gandiva
 
 # Install Java
 ARG java=1.8.0

ci/docker/python-wheel-manylinux-201x.dockerfile

@@ -53,49 +53,35 @@ COPY ci/vcpkg/*.patch \
 COPY ci/scripts/install_vcpkg.sh \
  ci/scripts/install_glibc.sh \
  arrow/ci/scripts/
-RUN arrow/ci/scripts/install_vcpkg.sh /opt/vcpkg ${vcpkg} && \
+ENV VCPKG_ROOT=/opt/vcpkg
+RUN arrow/ci/scripts/install_vcpkg.sh ${VCPKG_ROOT} ${vcpkg} && \
  if [ "${manylinux}" == "2010" ]; then \
  arrow/ci/scripts/install_glibc.sh ${glibc} /opt/glibc-${glibc} && \
- patchelf --set-interpreter /opt/glibc-2.18/lib/ld-linux-x86-64.so.2 /opt/vcpkg/vcpkg && \
- patchelf --set-rpath /opt/glibc-2.18/lib:/usr/lib64 /opt/vcpkg/vcpkg; \
+ patchelf --set-interpreter /opt/glibc-2.18/lib/ld-linux-x86-64.so.2 ${VCPKG_ROOT}/vcpkg && \
+ patchelf --set-rpath /opt/glibc-2.18/lib:/usr/lib64 ${VCPKG_ROOT}/vcpkg; \
  fi
-ENV PATH="/opt/vcpkg:${PATH}"
+ENV PATH="${PATH}:${VCPKG_ROOT}"
 
 ARG build_type=release
 ENV CMAKE_BUILD_TYPE=${build_type} \
  VCPKG_FORCE_SYSTEM_BINARIES=1 \
  VCPKG_OVERLAY_TRIPLETS=/arrow/ci/vcpkg \
  VCPKG_DEFAULT_TRIPLET=${arch_short}-linux-static-${build_type} \
- VCPKG_FEATURE_FLAGS=-manifests
-
-# Need to install the boost-build prior installing the boost packages, otherwise
-# vcpkg will raise an error.
-# TODO(kszucs): factor out the package enumeration to a text file and reuse it
-# from the windows image and potentially in a future macos wheel build
-RUN vcpkg install --clean-after-build \
- abseil \
- aws-sdk-cpp[config,cognito-identity,core,identity-management,s3,sts,transfer] \
- boost-filesystem \
- brotli \
- bzip2 \
- c-ares \
- curl \
- flatbuffers \
- gflags \
- glog \
- google-cloud-cpp[core,storage] \
- grpc \
- lz4 \
- openssl \
- orc \
- protobuf \
- rapidjson \
- re2 \
- snappy \
- thrift \
- utf8proc \
- zlib \
- zstd
+ VCPKG_FEATURE_FLAGS="manifests"
+COPY ci/vcpkg/vcpkg.json arrow/ci/vcpkg/
+# cannot use the S3 feature here because while aws-sdk-cpp=1.9.160 contains
+# ssl related fixies as well as we can patch the vcpkg portfile to support
+# arm machines it hits ARROW-15141 where we would need to fall back to 1.8.186
+# but we cannot patch those portfiles since vcpkg-tool handles the checkout of
+# previous versions => use bundled S3 build
+RUN vcpkg install \
+ --clean-after-build \
+ --x-install-root=${VCPKG_ROOT}/installed \
+ --x-manifest-root=/arrow/ci/vcpkg \
+ --x-feature=flight \
+ --x-feature=gcs \
+ --x-feature=json \
+ --x-feature=parquet
 
 ARG python=3.8
 ENV PYTHON_VERSION=${python}

ci/docker/python-wheel-windows-vs2017.dockerfile

@@ -36,8 +36,9 @@ COPY ci/vcpkg/*.patch \
  ci/vcpkg/*windows*.cmake \
  arrow/ci/vcpkg/
 COPY ci/scripts/install_vcpkg.sh arrow/ci/scripts/
+ENV VCPKG_ROOT=C:\\vcpkg
 RUN bash arrow/ci/scripts/install_vcpkg.sh /c/vcpkg %vcpkg% && \
- setx PATH "%PATH%;C:\vcpkg"
+ setx PATH "%PATH%;%VCPKG_ROOT%"
 
 # Configure vcpkg and install dependencies
 # NOTE: use windows batch environment notation for build arguments in RUN
@@ -48,34 +49,21 @@ ARG build_type=release
 ENV CMAKE_BUILD_TYPE=${build_type} \
  VCPKG_OVERLAY_TRIPLETS=C:\\arrow\\ci\\vcpkg \
  VCPKG_DEFAULT_TRIPLET=amd64-windows-static-md-${build_type} \
- VCPKG_FEATURE_FLAGS=-manifests
-
-RUN vcpkg install --clean-after-build \
- abseil \
- aws-sdk-cpp[config,cognito-identity,core,identity-management,s3,sts,transfer] \
- boost-filesystem \
- boost-multiprecision \
- boost-system \
- brotli \
- bzip2 \
- c-ares \
- curl \
- flatbuffers \
- gflags \
- glog \
- google-cloud-cpp[core,storage] \
- grpc \
- lz4 \
- openssl \
- orc \
- protobuf \
- rapidjson \
- re2 \
- snappy \
- thrift \
- utf8proc \
- zlib \
- zstd
+ VCPKG_FEATURE_FLAGS="manifests"
+COPY ci/vcpkg/vcpkg.json arrow/ci/vcpkg/
+# cannot use the S3 feature here because while aws-sdk-cpp=1.9.160 contains
+# ssl related fixies as well as we can patch the vcpkg portfile to support
+# arm machines it hits ARROW-15141 where we would need to fall back to 1.8.186
+# but we cannot patch those portfiles since vcpkg-tool handles the checkout of
+# previous versions => use bundled S3 build
+RUN vcpkg install \
+ --clean-after-build \
+ --x-install-root=%VCPKG_ROOT%\installed \
+ --x-manifest-root=arrow/ci/vcpkg \
+ --x-feature=flight \
+ --x-feature=gcs \
+ --x-feature=json \
+ --x-feature=parquet
 
 # Remove previous installations of python from the base image
 # NOTE: a more recent base image (tried with 2.12.1) comes with python 3.9.7
@@ -87,20 +75,20 @@ RUN wmic product where "name like 'python%%'" call uninstall /nointeractive && \
 
 # Define the full version number otherwise choco falls back to patch number 0 (3.7 => 3.7.0)
 ARG python=3.8
-RUN (if "%python%"=="3.7" setx PYTHON_VERSION 3.7.12) & \
- (if "%python%"=="3.8" setx PYTHON_VERSION 3.8.11) & \
- (if "%python%"=="3.9" setx PYTHON_VERSION 3.9.9) & \
- (if "%python%"=="3.10" setx PYTHON_VERSION 3.10.1)
+RUN (if "%python%"=="3.7" setx PYTHON_VERSION "3.7.9" && setx PATH "%PATH%;C:\Python37;C:\Python37\Scripts") & \
+ (if "%python%"=="3.8" setx PYTHON_VERSION "3.8.10" && setx PATH "%PATH%;C:\Python38;C:\Python38\Scripts") & \
+ (if "%python%"=="3.9" setx PYTHON_VERSION "3.9.7" && setx PATH "%PATH%;C:\Python39;C:\Python39\Scripts") & \
+ (if "%python%"=="3.10" setx PYTHON_VERSION "3.10.2" && setx PATH "%PATH%;C:\Python310;C:\Python310\Scripts")
 RUN choco install -r -y --no-progress python --version=%PYTHON_VERSION%
-RUN pip install -U pip
+RUN python -m pip install -U pip setuptools
 
 COPY python/requirements-wheel-build.txt arrow/python/
-RUN pip install -r arrow/python/requirements-wheel-build.txt
+RUN python -m pip install -r arrow/python/requirements-wheel-build.txt
 
-ENV CLCACHE_DIR="C:\clcache"
-ENV CLCACHE_COMPRESS=1
-ENV CLCACHE_COMPRESSLEVEL=6
-RUN pip install git+https://github.com/Nuitka/clcache.git
+# ENV CLCACHE_DIR="C:\clcache"
+# ENV CLCACHE_COMPRESS=1
+# ENV CLCACHE_COMPRESSLEVEL=6
+# RUN pip install git+https://github.com/Nuitka/clcache.git
 
 # For debugging purposes
 # RUN wget --no-check-certificate https://github.com/lucasg/Dependencies/releases/download/v1.10/Dependencies_x64_Release.zip

ci/scripts/java_jni_manylinux_build.sh

@@ -55,9 +55,6 @@ export ARROW_TEST_DATA="${arrow_dir}/testing/data"
 export PARQUET_TEST_DATA="${arrow_dir}/cpp/submodules/parquet-testing/data"
 export AWS_EC2_METADATA_DISABLED=TRUE
 
-# NOTE(kszucs): workaround for ARROW-15403 along with the ORC_* cmake variables
-vcpkg remove orc
-
 mkdir -p "${build_dir}"
 pushd "${build_dir}"
 

ci/scripts/python_wheel_macos_build.sh

@@ -87,16 +87,11 @@ echo "=== (${PYTHON_VERSION}) Building Arrow C++ libraries ==="
 : ${VCPKG_FEATURE_FLAGS:=-manifests}
 : ${VCPKG_TARGET_TRIPLET:=${VCPKG_DEFAULT_TRIPLET:-x64-osx-static-${CMAKE_BUILD_TYPE}}}
 
-# NOTE(kszucs): workaround for ARROW-15403 along with the ORC_* cmake variables
-vcpkg remove orc
-
 mkdir -p ${build_dir}/build
 pushd ${build_dir}/build
 
 cmake \
  -DARROW_BUILD_SHARED=ON \
- -DCMAKE_APPLE_SILICON_PROCESSOR=arm64 \
- -DCMAKE_OSX_ARCHITECTURES=${CMAKE_OSX_ARCHITECTURES} \
  -DARROW_BUILD_STATIC=OFF \
  -DARROW_BUILD_TESTS=OFF \
  -DARROW_DATASET=${ARROW_DATASET} \
@@ -124,13 +119,16 @@ cmake \
  -DARROW_WITH_SNAPPY=${ARROW_WITH_SNAPPY} \
  -DARROW_WITH_ZLIB=${ARROW_WITH_ZLIB} \
  -DARROW_WITH_ZSTD=${ARROW_WITH_ZSTD} \
+ -DAWSSDK_SOURCE=BUNDLED \
+ -DCMAKE_APPLE_SILICON_PROCESSOR=arm64 \
  -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} \
  -DCMAKE_INSTALL_LIBDIR=lib \
  -DCMAKE_INSTALL_PREFIX=${build_dir}/install \
+ -DCMAKE_OSX_ARCHITECTURES=${CMAKE_OSX_ARCHITECTURES} \
  -DCMAKE_UNITY_BUILD=${CMAKE_UNITY_BUILD} \
  -DOPENSSL_USE_STATIC_LIBS=ON \
- -DORC_SOURCE=BUNDLED \
  -DORC_PROTOBUF_EXECUTABLE=${VCPKG_ROOT}/installed/${VCPKG_TARGET_TRIPLET}/tools/protobuf/protoc \
+ -DORC_SOURCE=BUNDLED \
  -DVCPKG_MANIFEST_MODE=OFF \
  -DVCPKG_TARGET_TRIPLET=${VCPKG_TARGET_TRIPLET} \
  -G ${CMAKE_GENERATOR} \

ci/scripts/python_wheel_manylinux_build.sh

@@ -80,11 +80,9 @@ if [[ "$(uname -m)" == arm* ]] || [[ "$(uname -m)" == aarch* ]]; then
  export ARROW_EXTRA_CMAKE_FLAGS="-DARROW_JEMALLOC_LG_PAGE=16"
 fi
 
-# NOTE(kszucs): workaround for ARROW-15403 along with the ORC_* cmake variables
-vcpkg remove orc
-
 mkdir /tmp/arrow-build
 pushd /tmp/arrow-build
+
 cmake \
  -DARROW_BROTLI_USE_SHARED=OFF \
  -DARROW_BUILD_SHARED=ON \
@@ -115,13 +113,14 @@ cmake \
  -DARROW_WITH_SNAPPY=${ARROW_WITH_SNAPPY} \
  -DARROW_WITH_ZLIB=${ARROW_WITH_ZLIB} \
  -DARROW_WITH_ZSTD=${ARROW_WITH_ZSTD} \
+ -DAWSSDK_SOURCE=BUNDLED \
  -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} \
  -DCMAKE_INSTALL_LIBDIR=lib \
  -DCMAKE_INSTALL_PREFIX=/tmp/arrow-dist \
  -DCMAKE_UNITY_BUILD=${CMAKE_UNITY_BUILD} \
  -DOPENSSL_USE_STATIC_LIBS=ON \
- -DORC_SOURCE=BUNDLED \
  -DORC_PROTOBUF_EXECUTABLE=${VCPKG_ROOT}/installed/${VCPKG_TARGET_TRIPLET}/tools/protobuf/protoc \
+ -DORC_SOURCE=BUNDLED \
  -DVCPKG_MANIFEST_MODE=OFF \
  -DVCPKG_TARGET_TRIPLET=${VCPKG_TARGET_TRIPLET} \
  ${ARROW_EXTRA_CMAKE_FLAGS} \

ci/scripts/python_wheel_windows_build.bat

@@ -51,9 +51,6 @@ set VCPKG_ROOT=C:\vcpkg
 set VCPKG_FEATURE_FLAGS=-manifests
 set VCGPK_TARGET_TRIPLET=amd64-windows-static-md-%CMAKE_BUILD_TYPE%
 
-@rem NOTE(kszucs): workaround for ARROW-15403 along with the ORC_* cmake variables
-vcpkg remove orc
-
 mkdir C:\arrow-build
 pushd C:\arrow-build
 cmake ^
@@ -80,13 +77,12 @@ cmake ^
  -DARROW_WITH_SNAPPY=%ARROW_WITH_SNAPPY% ^
  -DARROW_WITH_ZLIB=%ARROW_WITH_ZLIB% ^
  -DARROW_WITH_ZSTD=%ARROW_WITH_ZSTD% ^
+ -DAWSSDK_SOURCE=BUNDLED ^
  -DCMAKE_BUILD_TYPE=%CMAKE_BUILD_TYPE% ^
  -DCMAKE_CXX_COMPILER=clcache ^
  -DCMAKE_INSTALL_PREFIX=C:\arrow-dist ^
  -DCMAKE_UNITY_BUILD=%CMAKE_UNITY_BUILD% ^
  -DMSVC_LINK_VERBOSE=ON ^
- -DORC_SOURCE=BUNDLED ^
- -DORC_PROTOBUF_EXECUTABLE=%VCPKG_ROOT%\installed\%VCGPK_TARGET_TRIPLET%\tools\protobuf\protoc.exe ^
  -DVCPKG_MANIFEST_MODE=OFF ^
  -DVCPKG_TARGET_TRIPLET=%VCGPK_TARGET_TRIPLET% ^
  -G "%CMAKE_GENERATOR%" ^

ci/scripts/python_wheel_windows_test.bat

@@ -41,15 +41,15 @@ pip install -r C:\arrow\python\requirements-wheel-test.txt || exit /B
 python -m pip install --no-index --find-links=C:\arrow\python\dist\ pyarrow || exit /B
 
 @REM Test that the modules are importable
-python -c "import pyarrow"
-python -c "import pyarrow._hdfs"
-python -c "import pyarrow._s3fs"
-python -c "import pyarrow.csv"
-python -c "import pyarrow.dataset"
-python -c "import pyarrow.flight"
-python -c "import pyarrow.fs"
-python -c "import pyarrow.json"
-python -c "import pyarrow.parquet"
+python -c "import pyarrow" || exit /B
+python -c "import pyarrow._hdfs" || exit /B
+python -c "import pyarrow._s3fs" || exit /B
+python -c "import pyarrow.csv" || exit /B
+python -c "import pyarrow.dataset" || exit /B
+python -c "import pyarrow.flight" || exit /B
+python -c "import pyarrow.fs" || exit /B
+python -c "import pyarrow.json" || exit /B
+python -c "import pyarrow.parquet" || exit /B
 
 @REM Execute unittest
 pytest -r s --pyargs pyarrow || exit /B

ci/vcpkg/ports.patch

@@ -164,20 +164,20 @@ index 00000000..4566cc66
 +- " set(LibCrypto_SHARED_LIBRARY ${OPENSSL_CRYPTO_LIBRARY} CACHE INTERNAL \"The OpenSSL crypto shared library\")\n"
 +- "endif()\n"
 + )
-+
++
 + if (NOT SIMPLE_INSTALL)
 diff --git a/ports/aws-sdk-cpp/portfile.cmake b/ports/aws-sdk-cpp/portfile.cmake
 index 2d6bba4d..0ac47887 100644
 --- a/ports/aws-sdk-cpp/portfile.cmake
 +++ b/ports/aws-sdk-cpp/portfile.cmake
 @@ -8,6 +8,7 @@ vcpkg_from_github(
- PATCHES
- patch-relocatable-rpath.patch
- fix-aws-root.patch
-+ fix-find-crypto.patch
- )
-
- string(COMPARE EQUAL "${VCPKG_CRT_LINKAGE}" "dynamic" FORCE_SHARED_CRT)
+ PATCHES
+ patch-relocatable-rpath.patch
+ fix-aws-root.patch
++ fix-find-crypto.patch
+ )
+
+ string(COMPARE EQUAL "${VCPKG_CRT_LINKAGE}" "dynamic" FORCE_SHARED_CRT)
 diff --git a/ports/aws-sdk-cpp/vcpkg.json b/ports/aws-sdk-cpp/vcpkg.json
 index 3836e2b6..cd88ef07 100644
 --- a/ports/aws-sdk-cpp/vcpkg.json
@@ -216,7 +216,7 @@ index 45b8c706..b409d8a7 100644
  HEAD_REF master
 + PATCHES "snappy-disable-bmi.patch"
  )
-
+
  vcpkg_cmake_configure(
 diff --git a/ports/snappy/snappy-disable-bmi.patch b/ports/snappy/snappy-disable-bmi.patch
 new file mode 100644
@@ -241,5 +241,5 @@ index 00000000..a57ce0c2
 + return v & ~(mask << (8 * n));
 +-#endif
 + }
-+
++
 + static inline bool LeftShiftOverflows(uint8_t value, uint32_t shift) {

ci/docker/java-jni-manylinux-201x.dockerfile

@@ -19,19 +19,16 @@ ARG base
 FROM ${base}
 
 # Install the libaries required by the Gandiva to run
-RUN vcpkg install --clean-after-build \
- boost-algorithm \
- boost-crc \
- boost-date-time \
- boost-format \
- boost-locale \
- boost-multiprecision \
- boost-predef \
- boost-regex \
- boost-system \
- boost-variant \
- # Use enable rtti to avoid link problems in Gandiva
- llvm[clang,default-options,default-targets,lld,tools,enable-rtti]
+# Use enable llvm[enable-rtti] in the vcpkg.json to avoid link problems in Gandiva
+RUN vcpkg install \
+ --clean-after-build \
+ --x-install-root=${VCPKG_ROOT}/installed \
+ --x-manifest-root=/arrow/ci/vcpkg \
+ --x-feature=flight \
+ --x-feature=gcs \
+ --x-feature=json \
+ --x-feature=parquet \
+ --x-feature=gandiva
 
 # Install Java
 ARG java=1.8.0

ci/docker/python-wheel-manylinux-201x.dockerfile

@@ -53,49 +53,35 @@ COPY ci/vcpkg/*.patch \
 COPY ci/scripts/install_vcpkg.sh \
  ci/scripts/install_glibc.sh \
  arrow/ci/scripts/
-RUN arrow/ci/scripts/install_vcpkg.sh /opt/vcpkg ${vcpkg} && \
+ENV VCPKG_ROOT=/opt/vcpkg
+RUN arrow/ci/scripts/install_vcpkg.sh ${VCPKG_ROOT} ${vcpkg} && \
  if [ "${manylinux}" == "2010" ]; then \
  arrow/ci/scripts/install_glibc.sh ${glibc} /opt/glibc-${glibc} && \
- patchelf --set-interpreter /opt/glibc-2.18/lib/ld-linux-x86-64.so.2 /opt/vcpkg/vcpkg && \
- patchelf --set-rpath /opt/glibc-2.18/lib:/usr/lib64 /opt/vcpkg/vcpkg; \
+ patchelf --set-interpreter /opt/glibc-2.18/lib/ld-linux-x86-64.so.2 ${VCPKG_ROOT}/vcpkg && \
+ patchelf --set-rpath /opt/glibc-2.18/lib:/usr/lib64 ${VCPKG_ROOT}/vcpkg; \
  fi
-ENV PATH="/opt/vcpkg:${PATH}"
+ENV PATH="${PATH}:${VCPKG_ROOT}"
 
 ARG build_type=release
 ENV CMAKE_BUILD_TYPE=${build_type} \
  VCPKG_FORCE_SYSTEM_BINARIES=1 \
  VCPKG_OVERLAY_TRIPLETS=/arrow/ci/vcpkg \
  VCPKG_DEFAULT_TRIPLET=${arch_short}-linux-static-${build_type} \
- VCPKG_FEATURE_FLAGS=-manifests
-
-# Need to install the boost-build prior installing the boost packages, otherwise
-# vcpkg will raise an error.
-# TODO(kszucs): factor out the package enumeration to a text file and reuse it
-# from the windows image and potentially in a future macos wheel build
-RUN vcpkg install --clean-after-build \
- abseil \
- aws-sdk-cpp[config,cognito-identity,core,identity-management,s3,sts,transfer] \
- boost-filesystem \
- brotli \
- bzip2 \
- c-ares \
- curl \
- flatbuffers \
- gflags \
- glog \
- google-cloud-cpp[core,storage] \
- grpc \
- lz4 \
- openssl \
- orc \
- protobuf \
- rapidjson \
- re2 \
- snappy \
- thrift \
- utf8proc \
- zlib \
- zstd
+ VCPKG_FEATURE_FLAGS="manifests"
+COPY ci/vcpkg/vcpkg.json arrow/ci/vcpkg/
+# cannot use the S3 feature here because while aws-sdk-cpp=1.9.160 contains
+# ssl related fixies as well as we can patch the vcpkg portfile to support
+# arm machines it hits ARROW-15141 where we would need to fall back to 1.8.186
+# but we cannot patch those portfiles since vcpkg-tool handles the checkout of
+# previous versions => use bundled S3 build
+RUN vcpkg install \
+ --clean-after-build \
+ --x-install-root=${VCPKG_ROOT}/installed \
+ --x-manifest-root=/arrow/ci/vcpkg \
+ --x-feature=flight \
+ --x-feature=gcs \
+ --x-feature=json \
+ --x-feature=parquet
 
 ARG python=3.8
 ENV PYTHON_VERSION=${python}

ci/docker/python-wheel-windows-vs2017.dockerfile

@@ -36,8 +36,9 @@ COPY ci/vcpkg/*.patch \
  ci/vcpkg/*windows*.cmake \
  arrow/ci/vcpkg/
 COPY ci/scripts/install_vcpkg.sh arrow/ci/scripts/
+ENV VCPKG_ROOT=C:\\vcpkg
 RUN bash arrow/ci/scripts/install_vcpkg.sh /c/vcpkg %vcpkg% && \
- setx PATH "%PATH%;C:\vcpkg"
+ setx PATH "%PATH%;%VCPKG_ROOT%"
 
 # Configure vcpkg and install dependencies
 # NOTE: use windows batch environment notation for build arguments in RUN
@@ -48,34 +49,21 @@ ARG build_type=release
 ENV CMAKE_BUILD_TYPE=${build_type} \
  VCPKG_OVERLAY_TRIPLETS=C:\\arrow\\ci\\vcpkg \
  VCPKG_DEFAULT_TRIPLET=amd64-windows-static-md-${build_type} \
- VCPKG_FEATURE_FLAGS=-manifests
-
-RUN vcpkg install --clean-after-build \
- abseil \
- aws-sdk-cpp[config,cognito-identity,core,identity-management,s3,sts,transfer] \
- boost-filesystem \
- boost-multiprecision \
- boost-system \
- brotli \
- bzip2 \
- c-ares \
- curl \
- flatbuffers \
- gflags \
- glog \
- google-cloud-cpp[core,storage] \
- grpc \
- lz4 \
- openssl \
- orc \
- protobuf \
- rapidjson \
- re2 \
- snappy \
- thrift \
- utf8proc \
- zlib \
- zstd
+ VCPKG_FEATURE_FLAGS="manifests"
+COPY ci/vcpkg/vcpkg.json arrow/ci/vcpkg/
+# cannot use the S3 feature here because while aws-sdk-cpp=1.9.160 contains
+# ssl related fixies as well as we can patch the vcpkg portfile to support
+# arm machines it hits ARROW-15141 where we would need to fall back to 1.8.186
+# but we cannot patch those portfiles since vcpkg-tool handles the checkout of
+# previous versions => use bundled S3 build
+RUN vcpkg install \
+ --clean-after-build \
+ --x-install-root=%VCPKG_ROOT%\installed \
+ --x-manifest-root=arrow/ci/vcpkg \
+ --x-feature=flight \
+ --x-feature=gcs \
+ --x-feature=json \
+ --x-feature=parquet
 
 # Remove previous installations of python from the base image
 # NOTE: a more recent base image (tried with 2.12.1) comes with python 3.9.7
@@ -87,20 +75,20 @@ RUN wmic product where "name like 'python%%'" call uninstall /nointeractive && \
 
 # Define the full version number otherwise choco falls back to patch number 0 (3.7 => 3.7.0)
 ARG python=3.8
-RUN (if "%python%"=="3.7" setx PYTHON_VERSION 3.7.12) & \
- (if "%python%"=="3.8" setx PYTHON_VERSION 3.8.11) & \
- (if "%python%"=="3.9" setx PYTHON_VERSION 3.9.9) & \
- (if "%python%"=="3.10" setx PYTHON_VERSION 3.10.1)
+RUN (if "%python%"=="3.7" setx PYTHON_VERSION "3.7.9" && setx PATH "%PATH%;C:\Python37;C:\Python37\Scripts") & \
+ (if "%python%"=="3.8" setx PYTHON_VERSION "3.8.10" && setx PATH "%PATH%;C:\Python38;C:\Python38\Scripts") & \
+ (if "%python%"=="3.9" setx PYTHON_VERSION "3.9.7" && setx PATH "%PATH%;C:\Python39;C:\Python39\Scripts") & \
+ (if "%python%"=="3.10" setx PYTHON_VERSION "3.10.2" && setx PATH "%PATH%;C:\Python310;C:\Python310\Scripts")
 RUN choco install -r -y --no-progress python --version=%PYTHON_VERSION%
-RUN pip install -U pip
+RUN python -m pip install -U pip setuptools
 
 COPY python/requirements-wheel-build.txt arrow/python/
-RUN pip install -r arrow/python/requirements-wheel-build.txt
+RUN python -m pip install -r arrow/python/requirements-wheel-build.txt
 
-ENV CLCACHE_DIR="C:\clcache"
-ENV CLCACHE_COMPRESS=1
-ENV CLCACHE_COMPRESSLEVEL=6
-RUN pip install git+https://github.com/Nuitka/clcache.git
+# ENV CLCACHE_DIR="C:\clcache"
+# ENV CLCACHE_COMPRESS=1
+# ENV CLCACHE_COMPRESSLEVEL=6
+# RUN pip install git+https://github.com/Nuitka/clcache.git
 
 # For debugging purposes
 # RUN wget --no-check-certificate https://github.com/lucasg/Dependencies/releases/download/v1.10/Dependencies_x64_Release.zip

ci/scripts/java_jni_manylinux_build.sh

@@ -55,9 +55,6 @@ export ARROW_TEST_DATA="${arrow_dir}/testing/data"
 export PARQUET_TEST_DATA="${arrow_dir}/cpp/submodules/parquet-testing/data"
 export AWS_EC2_METADATA_DISABLED=TRUE
 
-# NOTE(kszucs): workaround for ARROW-15403 along with the ORC_* cmake variables
-vcpkg remove orc
-
 mkdir -p "${build_dir}"
 pushd "${build_dir}"
 

ci/scripts/python_wheel_macos_build.sh

@@ -87,16 +87,11 @@ echo "=== (${PYTHON_VERSION}) Building Arrow C++ libraries ==="
 : ${VCPKG_FEATURE_FLAGS:=-manifests}
 : ${VCPKG_TARGET_TRIPLET:=${VCPKG_DEFAULT_TRIPLET:-x64-osx-static-${CMAKE_BUILD_TYPE}}}
 
-# NOTE(kszucs): workaround for ARROW-15403 along with the ORC_* cmake variables
-vcpkg remove orc
-
 mkdir -p ${build_dir}/build
 pushd ${build_dir}/build
 
 cmake \
  -DARROW_BUILD_SHARED=ON \
- -DCMAKE_APPLE_SILICON_PROCESSOR=arm64 \
- -DCMAKE_OSX_ARCHITECTURES=${CMAKE_OSX_ARCHITECTURES} \
  -DARROW_BUILD_STATIC=OFF \
  -DARROW_BUILD_TESTS=OFF \
  -DARROW_DATASET=${ARROW_DATASET} \
@@ -124,13 +119,16 @@ cmake \
  -DARROW_WITH_SNAPPY=${ARROW_WITH_SNAPPY} \
  -DARROW_WITH_ZLIB=${ARROW_WITH_ZLIB} \
  -DARROW_WITH_ZSTD=${ARROW_WITH_ZSTD} \
+ -DAWSSDK_SOURCE=BUNDLED \
+ -DCMAKE_APPLE_SILICON_PROCESSOR=arm64 \
  -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} \
  -DCMAKE_INSTALL_LIBDIR=lib \
  -DCMAKE_INSTALL_PREFIX=${build_dir}/install \
+ -DCMAKE_OSX_ARCHITECTURES=${CMAKE_OSX_ARCHITECTURES} \
  -DCMAKE_UNITY_BUILD=${CMAKE_UNITY_BUILD} \
  -DOPENSSL_USE_STATIC_LIBS=ON \
- -DORC_SOURCE=BUNDLED \
  -DORC_PROTOBUF_EXECUTABLE=${VCPKG_ROOT}/installed/${VCPKG_TARGET_TRIPLET}/tools/protobuf/protoc \
+ -DORC_SOURCE=BUNDLED \
  -DVCPKG_MANIFEST_MODE=OFF \
  -DVCPKG_TARGET_TRIPLET=${VCPKG_TARGET_TRIPLET} \
  -G ${CMAKE_GENERATOR} \

ci/scripts/python_wheel_manylinux_build.sh

@@ -80,11 +80,9 @@ if [[ "$(uname -m)" == arm* ]] || [[ "$(uname -m)" == aarch* ]]; then
  export ARROW_EXTRA_CMAKE_FLAGS="-DARROW_JEMALLOC_LG_PAGE=16"
 fi
 
-# NOTE(kszucs): workaround for ARROW-15403 along with the ORC_* cmake variables
-vcpkg remove orc
-
 mkdir /tmp/arrow-build
 pushd /tmp/arrow-build
+
 cmake \
  -DARROW_BROTLI_USE_SHARED=OFF \
  -DARROW_BUILD_SHARED=ON \
@@ -115,13 +113,14 @@ cmake \
  -DARROW_WITH_SNAPPY=${ARROW_WITH_SNAPPY} \
  -DARROW_WITH_ZLIB=${ARROW_WITH_ZLIB} \
  -DARROW_WITH_ZSTD=${ARROW_WITH_ZSTD} \
+ -DAWSSDK_SOURCE=BUNDLED \
  -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} \
  -DCMAKE_INSTALL_LIBDIR=lib \
  -DCMAKE_INSTALL_PREFIX=/tmp/arrow-dist \
  -DCMAKE_UNITY_BUILD=${CMAKE_UNITY_BUILD} \
  -DOPENSSL_USE_STATIC_LIBS=ON \
- -DORC_SOURCE=BUNDLED \
  -DORC_PROTOBUF_EXECUTABLE=${VCPKG_ROOT}/installed/${VCPKG_TARGET_TRIPLET}/tools/protobuf/protoc \
+ -DORC_SOURCE=BUNDLED \
  -DVCPKG_MANIFEST_MODE=OFF \
  -DVCPKG_TARGET_TRIPLET=${VCPKG_TARGET_TRIPLET} \
  ${ARROW_EXTRA_CMAKE_FLAGS} \

ci/scripts/python_wheel_windows_build.bat

@@ -51,9 +51,6 @@ set VCPKG_ROOT=C:\vcpkg
 set VCPKG_FEATURE_FLAGS=-manifests
 set VCGPK_TARGET_TRIPLET=amd64-windows-static-md-%CMAKE_BUILD_TYPE%
 
-@rem NOTE(kszucs): workaround for ARROW-15403 along with the ORC_* cmake variables
-vcpkg remove orc
-
 mkdir C:\arrow-build
 pushd C:\arrow-build
 cmake ^
@@ -80,13 +77,12 @@ cmake ^
  -DARROW_WITH_SNAPPY=%ARROW_WITH_SNAPPY% ^
  -DARROW_WITH_ZLIB=%ARROW_WITH_ZLIB% ^
  -DARROW_WITH_ZSTD=%ARROW_WITH_ZSTD% ^
+ -DAWSSDK_SOURCE=BUNDLED ^
  -DCMAKE_BUILD_TYPE=%CMAKE_BUILD_TYPE% ^
  -DCMAKE_CXX_COMPILER=clcache ^
  -DCMAKE_INSTALL_PREFIX=C:\arrow-dist ^
  -DCMAKE_UNITY_BUILD=%CMAKE_UNITY_BUILD% ^
  -DMSVC_LINK_VERBOSE=ON ^
- -DORC_SOURCE=BUNDLED ^
- -DORC_PROTOBUF_EXECUTABLE=%VCPKG_ROOT%\installed\%VCGPK_TARGET_TRIPLET%\tools\protobuf\protoc.exe ^
  -DVCPKG_MANIFEST_MODE=OFF ^
  -DVCPKG_TARGET_TRIPLET=%VCGPK_TARGET_TRIPLET% ^
  -G "%CMAKE_GENERATOR%" ^

ci/scripts/python_wheel_windows_test.bat

@@ -41,15 +41,15 @@ pip install -r C:\arrow\python\requirements-wheel-test.txt || exit /B
 python -m pip install --no-index --find-links=C:\arrow\python\dist\ pyarrow || exit /B
 
 @REM Test that the modules are importable
-python -c "import pyarrow"
-python -c "import pyarrow._hdfs"
-python -c "import pyarrow._s3fs"
-python -c "import pyarrow.csv"
-python -c "import pyarrow.dataset"
-python -c "import pyarrow.flight"
-python -c "import pyarrow.fs"
-python -c "import pyarrow.json"
-python -c "import pyarrow.parquet"
+python -c "import pyarrow" || exit /B
+python -c "import pyarrow._hdfs" || exit /B
+python -c "import pyarrow._s3fs" || exit /B
+python -c "import pyarrow.csv" || exit /B
+python -c "import pyarrow.dataset" || exit /B
+python -c "import pyarrow.flight" || exit /B
+python -c "import pyarrow.fs" || exit /B
+python -c "import pyarrow.json" || exit /B
+python -c "import pyarrow.parquet" || exit /B
 
 @REM Execute unittest
 pytest -r s --pyargs pyarrow || exit /B

ci/vcpkg/ports.patch

@@ -164,20 +164,20 @@ index 00000000..4566cc66
 +- " set(LibCrypto_SHARED_LIBRARY ${OPENSSL_CRYPTO_LIBRARY} CACHE INTERNAL \"The OpenSSL crypto shared library\")\n"
 +- "endif()\n"
 + )
-+
++
 + if (NOT SIMPLE_INSTALL)
 diff --git a/ports/aws-sdk-cpp/portfile.cmake b/ports/aws-sdk-cpp/portfile.cmake
 index 2d6bba4d..0ac47887 100644
 --- a/ports/aws-sdk-cpp/portfile.cmake
 +++ b/ports/aws-sdk-cpp/portfile.cmake
 @@ -8,6 +8,7 @@ vcpkg_from_github(
- PATCHES
- patch-relocatable-rpath.patch
- fix-aws-root.patch
-+ fix-find-crypto.patch
- )
-
- string(COMPARE EQUAL "${VCPKG_CRT_LINKAGE}" "dynamic" FORCE_SHARED_CRT)
+ PATCHES
+ patch-relocatable-rpath.patch
+ fix-aws-root.patch
++ fix-find-crypto.patch
+ )
+
+ string(COMPARE EQUAL "${VCPKG_CRT_LINKAGE}" "dynamic" FORCE_SHARED_CRT)
 diff --git a/ports/aws-sdk-cpp/vcpkg.json b/ports/aws-sdk-cpp/vcpkg.json
 index 3836e2b6..cd88ef07 100644
 --- a/ports/aws-sdk-cpp/vcpkg.json
@@ -216,7 +216,7 @@ index 45b8c706..b409d8a7 100644
  HEAD_REF master
 + PATCHES "snappy-disable-bmi.patch"
  )
-
+
  vcpkg_cmake_configure(
 diff --git a/ports/snappy/snappy-disable-bmi.patch b/ports/snappy/snappy-disable-bmi.patch
 new file mode 100644
@@ -241,5 +241,5 @@ index 00000000..a57ce0c2
 + return v & ~(mask << (8 * n));
 +-#endif
 + }
-+
++
 + static inline bool LeftShiftOverflows(uint8_t value, uint32_t shift) {

ci/docker/java-jni-manylinux-201x.dockerfile

@@ -19,19 +19,16 @@ ARG base
 FROM ${base}
 
 # Install the libaries required by the Gandiva to run
-RUN vcpkg install --clean-after-build \
- boost-algorithm \
- boost-crc \
- boost-date-time \
- boost-format \
- boost-locale \
- boost-multiprecision \
- boost-predef \
- boost-regex \
- boost-system \
- boost-variant \
- # Use enable rtti to avoid link problems in Gandiva
- llvm[clang,default-options,default-targets,lld,tools,enable-rtti]
+# Use enable llvm[enable-rtti] in the vcpkg.json to avoid link problems in Gandiva
+RUN vcpkg install \
+ --clean-after-build \
+ --x-install-root=${VCPKG_ROOT}/installed \
+ --x-manifest-root=/arrow/ci/vcpkg \
+ --x-feature=flight \
+ --x-feature=gcs \
+ --x-feature=json \
+ --x-feature=parquet \
+ --x-feature=gandiva
 
 # Install Java
 ARG java=1.8.0

ci/docker/python-wheel-manylinux-201x.dockerfile

@@ -53,49 +53,35 @@ COPY ci/vcpkg/*.patch \
 COPY ci/scripts/install_vcpkg.sh \
  ci/scripts/install_glibc.sh \
  arrow/ci/scripts/
-RUN arrow/ci/scripts/install_vcpkg.sh /opt/vcpkg ${vcpkg} && \
+ENV VCPKG_ROOT=/opt/vcpkg
+RUN arrow/ci/scripts/install_vcpkg.sh ${VCPKG_ROOT} ${vcpkg} && \
  if [ "${manylinux}" == "2010" ]; then \
  arrow/ci/scripts/install_glibc.sh ${glibc} /opt/glibc-${glibc} && \
- patchelf --set-interpreter /opt/glibc-2.18/lib/ld-linux-x86-64.so.2 /opt/vcpkg/vcpkg && \
- patchelf --set-rpath /opt/glibc-2.18/lib:/usr/lib64 /opt/vcpkg/vcpkg; \
+ patchelf --set-interpreter /opt/glibc-2.18/lib/ld-linux-x86-64.so.2 ${VCPKG_ROOT}/vcpkg && \
+ patchelf --set-rpath /opt/glibc-2.18/lib:/usr/lib64 ${VCPKG_ROOT}/vcpkg; \
  fi
-ENV PATH="/opt/vcpkg:${PATH}"
+ENV PATH="${PATH}:${VCPKG_ROOT}"
 
 ARG build_type=release
 ENV CMAKE_BUILD_TYPE=${build_type} \
  VCPKG_FORCE_SYSTEM_BINARIES=1 \
  VCPKG_OVERLAY_TRIPLETS=/arrow/ci/vcpkg \
  VCPKG_DEFAULT_TRIPLET=${arch_short}-linux-static-${build_type} \
- VCPKG_FEATURE_FLAGS=-manifests
-
-# Need to install the boost-build prior installing the boost packages, otherwise
-# vcpkg will raise an error.
-# TODO(kszucs): factor out the package enumeration to a text file and reuse it
-# from the windows image and potentially in a future macos wheel build
-RUN vcpkg install --clean-after-build \
- abseil \
- aws-sdk-cpp[config,cognito-identity,core,identity-management,s3,sts,transfer] \
- boost-filesystem \
- brotli \
- bzip2 \
- c-ares \
- curl \
- flatbuffers \
- gflags \
- glog \
- google-cloud-cpp[core,storage] \
- grpc \
- lz4 \
- openssl \
- orc \
- protobuf \
- rapidjson \
- re2 \
- snappy \
- thrift \
- utf8proc \
- zlib \
- zstd
+ VCPKG_FEATURE_FLAGS="manifests"
+COPY ci/vcpkg/vcpkg.json arrow/ci/vcpkg/
+# cannot use the S3 feature here because while aws-sdk-cpp=1.9.160 contains
+# ssl related fixies as well as we can patch the vcpkg portfile to support
+# arm machines it hits ARROW-15141 where we would need to fall back to 1.8.186
+# but we cannot patch those portfiles since vcpkg-tool handles the checkout of
+# previous versions => use bundled S3 build
+RUN vcpkg install \
+ --clean-after-build \
+ --x-install-root=${VCPKG_ROOT}/installed \
+ --x-manifest-root=/arrow/ci/vcpkg \
+ --x-feature=flight \
+ --x-feature=gcs \
+ --x-feature=json \
+ --x-feature=parquet
 
 ARG python=3.8
 ENV PYTHON_VERSION=${python}

ci/docker/python-wheel-windows-vs2017.dockerfile

@@ -36,8 +36,9 @@ COPY ci/vcpkg/*.patch \
  ci/vcpkg/*windows*.cmake \
  arrow/ci/vcpkg/
 COPY ci/scripts/install_vcpkg.sh arrow/ci/scripts/
+ENV VCPKG_ROOT=C:\\vcpkg
 RUN bash arrow/ci/scripts/install_vcpkg.sh /c/vcpkg %vcpkg% && \
- setx PATH "%PATH%;C:\vcpkg"
+ setx PATH "%PATH%;%VCPKG_ROOT%"
 
 # Configure vcpkg and install dependencies
 # NOTE: use windows batch environment notation for build arguments in RUN
@@ -48,34 +49,21 @@ ARG build_type=release
 ENV CMAKE_BUILD_TYPE=${build_type} \
  VCPKG_OVERLAY_TRIPLETS=C:\\arrow\\ci\\vcpkg \
  VCPKG_DEFAULT_TRIPLET=amd64-windows-static-md-${build_type} \
- VCPKG_FEATURE_FLAGS=-manifests
-
-RUN vcpkg install --clean-after-build \
- abseil \
- aws-sdk-cpp[config,cognito-identity,core,identity-management,s3,sts,transfer] \
- boost-filesystem \
- boost-multiprecision \
- boost-system \
- brotli \
- bzip2 \
- c-ares \
- curl \
- flatbuffers \
- gflags \
- glog \
- google-cloud-cpp[core,storage] \
- grpc \
- lz4 \
- openssl \
- orc \
- protobuf \
- rapidjson \
- re2 \
- snappy \
- thrift \
- utf8proc \
- zlib \
- zstd
+ VCPKG_FEATURE_FLAGS="manifests"
+COPY ci/vcpkg/vcpkg.json arrow/ci/vcpkg/
+# cannot use the S3 feature here because while aws-sdk-cpp=1.9.160 contains
+# ssl related fixies as well as we can patch the vcpkg portfile to support
+# arm machines it hits ARROW-15141 where we would need to fall back to 1.8.186
+# but we cannot patch those portfiles since vcpkg-tool handles the checkout of
+# previous versions => use bundled S3 build
+RUN vcpkg install \
+ --clean-after-build \
+ --x-install-root=%VCPKG_ROOT%\installed \
+ --x-manifest-root=arrow/ci/vcpkg \
+ --x-feature=flight \
+ --x-feature=gcs \
+ --x-feature=json \
+ --x-feature=parquet
 
 # Remove previous installations of python from the base image
 # NOTE: a more recent base image (tried with 2.12.1) comes with python 3.9.7
@@ -87,20 +75,20 @@ RUN wmic product where "name like 'python%%'" call uninstall /nointeractive && \
 
 # Define the full version number otherwise choco falls back to patch number 0 (3.7 => 3.7.0)
 ARG python=3.8
-RUN (if "%python%"=="3.7" setx PYTHON_VERSION 3.7.12) & \
- (if "%python%"=="3.8" setx PYTHON_VERSION 3.8.11) & \
- (if "%python%"=="3.9" setx PYTHON_VERSION 3.9.9) & \
- (if "%python%"=="3.10" setx PYTHON_VERSION 3.10.1)
+RUN (if "%python%"=="3.7" setx PYTHON_VERSION "3.7.9" && setx PATH "%PATH%;C:\Python37;C:\Python37\Scripts") & \
+ (if "%python%"=="3.8" setx PYTHON_VERSION "3.8.10" && setx PATH "%PATH%;C:\Python38;C:\Python38\Scripts") & \
+ (if "%python%"=="3.9" setx PYTHON_VERSION "3.9.7" && setx PATH "%PATH%;C:\Python39;C:\Python39\Scripts") & \
+ (if "%python%"=="3.10" setx PYTHON_VERSION "3.10.2" && setx PATH "%PATH%;C:\Python310;C:\Python310\Scripts")
 RUN choco install -r -y --no-progress python --version=%PYTHON_VERSION%
-RUN pip install -U pip
+RUN python -m pip install -U pip setuptools
 
 COPY python/requirements-wheel-build.txt arrow/python/
-RUN pip install -r arrow/python/requirements-wheel-build.txt
+RUN python -m pip install -r arrow/python/requirements-wheel-build.txt
 
-ENV CLCACHE_DIR="C:\clcache"
-ENV CLCACHE_COMPRESS=1
-ENV CLCACHE_COMPRESSLEVEL=6
-RUN pip install git+https://github.com/Nuitka/clcache.git
+# ENV CLCACHE_DIR="C:\clcache"
+# ENV CLCACHE_COMPRESS=1
+# ENV CLCACHE_COMPRESSLEVEL=6
+# RUN pip install git+https://github.com/Nuitka/clcache.git
 
 # For debugging purposes
 # RUN wget --no-check-certificate https://github.com/lucasg/Dependencies/releases/download/v1.10/Dependencies_x64_Release.zip

ci/scripts/java_jni_manylinux_build.sh

@@ -55,9 +55,6 @@ export ARROW_TEST_DATA="${arrow_dir}/testing/data"
 export PARQUET_TEST_DATA="${arrow_dir}/cpp/submodules/parquet-testing/data"
 export AWS_EC2_METADATA_DISABLED=TRUE
 
-# NOTE(kszucs): workaround for ARROW-15403 along with the ORC_* cmake variables
-vcpkg remove orc
-
 mkdir -p "${build_dir}"
 pushd "${build_dir}"
 

ci/scripts/python_wheel_macos_build.sh

@@ -87,16 +87,11 @@ echo "=== (${PYTHON_VERSION}) Building Arrow C++ libraries ==="
 : ${VCPKG_FEATURE_FLAGS:=-manifests}
 : ${VCPKG_TARGET_TRIPLET:=${VCPKG_DEFAULT_TRIPLET:-x64-osx-static-${CMAKE_BUILD_TYPE}}}
 
-# NOTE(kszucs): workaround for ARROW-15403 along with the ORC_* cmake variables
-vcpkg remove orc
-
 mkdir -p ${build_dir}/build
 pushd ${build_dir}/build
 
 cmake \
  -DARROW_BUILD_SHARED=ON \
- -DCMAKE_APPLE_SILICON_PROCESSOR=arm64 \
- -DCMAKE_OSX_ARCHITECTURES=${CMAKE_OSX_ARCHITECTURES} \
  -DARROW_BUILD_STATIC=OFF \
  -DARROW_BUILD_TESTS=OFF \
  -DARROW_DATASET=${ARROW_DATASET} \
@@ -124,13 +119,16 @@ cmake \
  -DARROW_WITH_SNAPPY=${ARROW_WITH_SNAPPY} \
  -DARROW_WITH_ZLIB=${ARROW_WITH_ZLIB} \
  -DARROW_WITH_ZSTD=${ARROW_WITH_ZSTD} \
+ -DAWSSDK_SOURCE=BUNDLED \
+ -DCMAKE_APPLE_SILICON_PROCESSOR=arm64 \
  -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} \
  -DCMAKE_INSTALL_LIBDIR=lib \
  -DCMAKE_INSTALL_PREFIX=${build_dir}/install \
+ -DCMAKE_OSX_ARCHITECTURES=${CMAKE_OSX_ARCHITECTURES} \
  -DCMAKE_UNITY_BUILD=${CMAKE_UNITY_BUILD} \
  -DOPENSSL_USE_STATIC_LIBS=ON \
- -DORC_SOURCE=BUNDLED \
  -DORC_PROTOBUF_EXECUTABLE=${VCPKG_ROOT}/installed/${VCPKG_TARGET_TRIPLET}/tools/protobuf/protoc \
+ -DORC_SOURCE=BUNDLED \
  -DVCPKG_MANIFEST_MODE=OFF \
  -DVCPKG_TARGET_TRIPLET=${VCPKG_TARGET_TRIPLET} \
  -G ${CMAKE_GENERATOR} \

ci/scripts/python_wheel_manylinux_build.sh

@@ -80,11 +80,9 @@ if [[ "$(uname -m)" == arm* ]] || [[ "$(uname -m)" == aarch* ]]; then
  export ARROW_EXTRA_CMAKE_FLAGS="-DARROW_JEMALLOC_LG_PAGE=16"
 fi
 
-# NOTE(kszucs): workaround for ARROW-15403 along with the ORC_* cmake variables
-vcpkg remove orc
-
 mkdir /tmp/arrow-build
 pushd /tmp/arrow-build
+
 cmake \
  -DARROW_BROTLI_USE_SHARED=OFF \
  -DARROW_BUILD_SHARED=ON \
@@ -115,13 +113,14 @@ cmake \
  -DARROW_WITH_SNAPPY=${ARROW_WITH_SNAPPY} \
  -DARROW_WITH_ZLIB=${ARROW_WITH_ZLIB} \
  -DARROW_WITH_ZSTD=${ARROW_WITH_ZSTD} \
+ -DAWSSDK_SOURCE=BUNDLED \
  -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} \
  -DCMAKE_INSTALL_LIBDIR=lib \
  -DCMAKE_INSTALL_PREFIX=/tmp/arrow-dist \
  -DCMAKE_UNITY_BUILD=${CMAKE_UNITY_BUILD} \
  -DOPENSSL_USE_STATIC_LIBS=ON \
- -DORC_SOURCE=BUNDLED \
  -DORC_PROTOBUF_EXECUTABLE=${VCPKG_ROOT}/installed/${VCPKG_TARGET_TRIPLET}/tools/protobuf/protoc \
+ -DORC_SOURCE=BUNDLED \
  -DVCPKG_MANIFEST_MODE=OFF \
  -DVCPKG_TARGET_TRIPLET=${VCPKG_TARGET_TRIPLET} \
  ${ARROW_EXTRA_CMAKE_FLAGS} \

ci/scripts/python_wheel_windows_build.bat

@@ -51,9 +51,6 @@ set VCPKG_ROOT=C:\vcpkg
 set VCPKG_FEATURE_FLAGS=-manifests
 set VCGPK_TARGET_TRIPLET=amd64-windows-static-md-%CMAKE_BUILD_TYPE%
 
-@rem NOTE(kszucs): workaround for ARROW-15403 along with the ORC_* cmake variables
-vcpkg remove orc
-
 mkdir C:\arrow-build
 pushd C:\arrow-build
 cmake ^
@@ -80,13 +77,12 @@ cmake ^
  -DARROW_WITH_SNAPPY=%ARROW_WITH_SNAPPY% ^
  -DARROW_WITH_ZLIB=%ARROW_WITH_ZLIB% ^
  -DARROW_WITH_ZSTD=%ARROW_WITH_ZSTD% ^
+ -DAWSSDK_SOURCE=BUNDLED ^
  -DCMAKE_BUILD_TYPE=%CMAKE_BUILD_TYPE% ^
  -DCMAKE_CXX_COMPILER=clcache ^
  -DCMAKE_INSTALL_PREFIX=C:\arrow-dist ^
  -DCMAKE_UNITY_BUILD=%CMAKE_UNITY_BUILD% ^
  -DMSVC_LINK_VERBOSE=ON ^
- -DORC_SOURCE=BUNDLED ^
- -DORC_PROTOBUF_EXECUTABLE=%VCPKG_ROOT%\installed\%VCGPK_TARGET_TRIPLET%\tools\protobuf\protoc.exe ^
  -DVCPKG_MANIFEST_MODE=OFF ^
  -DVCPKG_TARGET_TRIPLET=%VCGPK_TARGET_TRIPLET% ^
  -G "%CMAKE_GENERATOR%" ^

ci/scripts/python_wheel_windows_test.bat

@@ -41,15 +41,15 @@ pip install -r C:\arrow\python\requirements-wheel-test.txt || exit /B
 python -m pip install --no-index --find-links=C:\arrow\python\dist\ pyarrow || exit /B
 
 @REM Test that the modules are importable
-python -c "import pyarrow"
-python -c "import pyarrow._hdfs"
-python -c "import pyarrow._s3fs"
-python -c "import pyarrow.csv"
-python -c "import pyarrow.dataset"
-python -c "import pyarrow.flight"
-python -c "import pyarrow.fs"
-python -c "import pyarrow.json"
-python -c "import pyarrow.parquet"
+python -c "import pyarrow" || exit /B
+python -c "import pyarrow._hdfs" || exit /B
+python -c "import pyarrow._s3fs" || exit /B
+python -c "import pyarrow.csv" || exit /B
+python -c "import pyarrow.dataset" || exit /B
+python -c "import pyarrow.flight" || exit /B
+python -c "import pyarrow.fs" || exit /B
+python -c "import pyarrow.json" || exit /B
+python -c "import pyarrow.parquet" || exit /B
 
 @REM Execute unittest
 pytest -r s --pyargs pyarrow || exit /B

ci/vcpkg/ports.patch

@@ -164,20 +164,20 @@ index 00000000..4566cc66
 +- " set(LibCrypto_SHARED_LIBRARY ${OPENSSL_CRYPTO_LIBRARY} CACHE INTERNAL \"The OpenSSL crypto shared library\")\n"
 +- "endif()\n"
 + )
-+
++
 + if (NOT SIMPLE_INSTALL)
 diff --git a/ports/aws-sdk-cpp/portfile.cmake b/ports/aws-sdk-cpp/portfile.cmake
 index 2d6bba4d..0ac47887 100644
 --- a/ports/aws-sdk-cpp/portfile.cmake
 +++ b/ports/aws-sdk-cpp/portfile.cmake
 @@ -8,6 +8,7 @@ vcpkg_from_github(
- PATCHES
- patch-relocatable-rpath.patch
- fix-aws-root.patch
-+ fix-find-crypto.patch
- )
-
- string(COMPARE EQUAL "${VCPKG_CRT_LINKAGE}" "dynamic" FORCE_SHARED_CRT)
+ PATCHES
+ patch-relocatable-rpath.patch
+ fix-aws-root.patch
++ fix-find-crypto.patch
+ )
+
+ string(COMPARE EQUAL "${VCPKG_CRT_LINKAGE}" "dynamic" FORCE_SHARED_CRT)
 diff --git a/ports/aws-sdk-cpp/vcpkg.json b/ports/aws-sdk-cpp/vcpkg.json
 index 3836e2b6..cd88ef07 100644
 --- a/ports/aws-sdk-cpp/vcpkg.json
@@ -216,7 +216,7 @@ index 45b8c706..b409d8a7 100644
  HEAD_REF master
 + PATCHES "snappy-disable-bmi.patch"
  )
-
+
  vcpkg_cmake_configure(
 diff --git a/ports/snappy/snappy-disable-bmi.patch b/ports/snappy/snappy-disable-bmi.patch
 new file mode 100644
@@ -241,5 +241,5 @@ index 00000000..a57ce0c2
 + return v & ~(mask << (8 * n));
 +-#endif
 + }
-+
++
 + static inline bool LeftShiftOverflows(uint8_t value, uint32_t shift) {