ARROW-6920: [Packaging] Build python 3.8 wheels

cpp/src/arrow/python/CMakeLists.txt

@@ -52,10 +52,9 @@ if(CMAKE_CXX_COMPILER_ID STREQUAL "AppleClang" OR CMAKE_CXX_COMPILER_ID STREQUAL
  set_property(SOURCE pyarrow.cc APPEND_STRING PROPERTY COMPILE_FLAGS " -Wno-cast-qual ")
 endif()
 
-set(ARROW_PYTHON_SHARED_LINK_LIBS arrow_shared ${PYTHON_OTHER_LIBS})
-
+set(ARROW_PYTHON_SHARED_LINK_LIBS arrow_shared)
 if(WIN32)
-set(ARROW_PYTHON_SHARED_LINK_LIBS ${ARROW_PYTHON_SHARED_LINK_LIBS} ${PYTHON_LIBRARIES})
+list(APPEND ARROW_PYTHON_SHARED_LINK_LIBS ${PYTHON_LIBRARIES} ${PYTHON_OTHER_LIBS})
 endif()
 
 set(ARROW_PYTHON_INCLUDES ${NUMPY_INCLUDE_DIRS} ${PYTHON_INCLUDE_DIRS})

dev/tasks/python-wheels/azure.linux.yml

@@ -0,0 +1,92 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+jobs:
+- job: linux
+ pool:
+ vmImage: ubuntu-latest
+ timeoutInMinutes: 360
+ steps:
+ - task: CondaEnvironment@1
+ inputs:
+ packageSpecs: 'click github3.py jinja2 jira pygit2 ruamel.yaml setuptools_scm toolz'
+ installOptions: '-c conda-forge'
+ updateConda: false
+
+ - script: |
+ set -ex
+ git clone --no-checkout {{ arrow.remote }} arrow
+ git -C arrow fetch -t {{ arrow.remote }} {{ arrow.branch }}
+ git -C arrow checkout FETCH_HEAD
+ git -C arrow submodule update --init --recursive
+ displayName: Clone arrow
+
+ - script: |
+ set -ex
+ docker -v
+ docker-compose -v
+ cd arrow
+ BUILD_IMAGE=centos-python-{{ wheel_tag }}
+ docker-compose pull $BUILD_IMAGE
+ # TODO(kou): Uncomment this when we resolve "ADD never use cache" problem.
+ # docker-compose build $BUILD_IMAGE
+ docker-compose run \
+ -e SETUPTOOLS_SCM_PRETEND_VERSION={{ arrow.no_rc_version }} \
+ -e PYTHON_VERSION="{{ python_version }}" \
+ -e UNICODE_WIDTH="{{ unicode_width }}" \
+ $BUILD_IMAGE
+ displayName: Build wheel
+
+ # auditwheel does always exit with 0 so it is mostly for debugging
+ # purposes
+ - script: |
+ set -ex
+ cd arrow
+ docker run \
+ -v $(pwd):/arrow quay.io/pypa/{{ wheel_tag }}_x86_64 \
+ /bin/bash -c \
+ "auditwheel show /arrow/python/{{ wheel_tag }}/dist/*.whl"
+ displayName: Audit wheel
+
+ - script: |
+ set -ex
+ cd arrow
+ test_args=""
+ {%- if test_remove_system_libs %}
+ test_args="${test_args} --remove-system-libs"
+ {%- endif %}
+ {%- for image in test_docker_images %}
+ docker run \
+ --shm-size 2G \
+ -v $(pwd):/arrow \
+ -e WHEEL_TAG="{{ wheel_tag }}" \
+ {{ image }} \
+ /arrow/dev/tasks/python-wheels/manylinux-test.sh ${test_args}
+ {%- endfor %}
+ displayName: Test wheel
+
+ - script: |
+ set -ex
+ python arrow/dev/tasks/crossbow.py \
+ --queue-path . \
+ --queue-remote {{ queue.remote_url }} \
+ upload-artifacts \
+ --pattern "arrow/python/{{ wheel_tag }}/dist/*" \
+ --sha {{ task.branch }} \
+ --tag {{ task.tag }}
+ env:
+ CROSSBOW_GITHUB_TOKEN: $(CROSSBOW_GITHUB_TOKEN)
+ displayName: Upload packages as a GitHub release

dev/tasks/python-wheels/travis.osx.yml

@@ -31,8 +31,8 @@ env:
  - MACOSX_DEPLOYMENT_TARGET="10.9"
 
 before_install:
- - git clone https://github.com/matthew-brett/multibuild # TODO pin it
- - git -C multibuild checkout 4e7a9396e9a50731bb83fc0d16bb98fb0c4032d7
+ - git clone https://github.com/matthew-brett/multibuild
+ - git -C multibuild checkout 68a4af043e2adb0d9353d4a0e1f3d871203237aa
 
  - git clone --no-checkout {{ arrow.remote }} arrow
  - git -C arrow fetch -t {{ arrow.remote }} {{ arrow.branch }}

dev/tasks/python-wheels/win-build.bat

@@ -20,14 +20,14 @@
 @rem create conda environment for compiling
 conda update --yes --quiet conda
 
-call conda create -n wheel-build -q -y -c conda-forge ^
+conda create -n wheel-build -q -y -c conda-forge ^
  --file=%ARROW_SRC%\ci\conda_env_cpp.yml ^
  --file=%ARROW_SRC%\ci\conda_env_gandiva.yml ^
  python=%PYTHON_VERSION% ^
  numpy=%NUMPY_VERSION% ^
  || exit /B
 
-call activate wheel-build
+call conda.bat activate wheel-build
 
 @rem Cannot use conda_env_python.yml here because conda-forge has
 @rem ceased providing up-to-date packages for Python 3.5
@@ -84,17 +84,17 @@ pushd %ARROW_SRC%\python
 python setup.py bdist_wheel || exit /B
 popd
 
-call deactivate
+call conda.bat deactivate
 
 set ARROW_TEST_DATA=%ARROW_SRC%\testing\data
 
 @rem test the wheel
 @rem TODO For maximum reliability, we should test in a plain virtualenv instead.
-call conda create -n wheel-test -c conda-forge -q -y ^
+conda create -n wheel-test -c conda-forge -q -y ^
  --file %ARROW_SRC%\ci\conda_env_python.yml ^
  python=%PYTHON_VERSION% ^
  numpy=%NUMPY_VERSION% || exit /B
-call activate wheel-test
+call conda.bat activate wheel-test
 
 @rem install the built wheel
 pip install -vv --no-index --find-links=%ARROW_SRC%\python\dist\ pyarrow || exit /B