GH-40454: [CI][Debian] Update Debian to 12 from 11#40455

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
raulcd merged 4 commits into from
Mar 13, 2024
Merged

GH-40454: [CI][Debian] Update Debian to 12 from 11 #40455

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
281c437
GH-40454: [CI][Debian] Update Debian to 12 from 11
kou Mar 11, 2024
3a5b568
Update pip
kou Mar 11, 2024
429fcda
Free space
kou Mar 12, 2024
8039954
Add missing $
kou Mar 12, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .env
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -48,7 +48,7 @@ ULIMIT_CORE=-1
# Default versions for platforms
ALMALINUX=8
ALPINE_LINUX=3.16
DEBIAN=11
DEBIAN=12
FEDORA=39
UBUNTU=20.04

Expand Down
8 changes: 4 additions & 4 deletions .github/workflows/go.yml
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -90,7 +90,7 @@ jobs:
echo "JSON" >> "$GITHUB_OUTPUT"

docker:
name: ${{ matrix.arch-label }} Debian 11 Go ${{ matrix.go }}
name: ${{ matrix.arch-label }} Debian 12 Go ${{ matrix.go }}
needs: docker-targets
runs-on: ${{ matrix.runs-on }}
if: ${{ !contains(github.event.pull_request.title, 'WIP') }}
Expand DownExpand Up@@ -154,7 +154,7 @@ jobs:
CONBENCH_EMAIL: ${{ secrets.CONBENCH_EMAIL }}
CONBENCH_PASSWORD: ${{ secrets.CONBENCH_PASS }}
CONBENCH_REF: ${{ github.ref_name }}
CONBENCH_MACHINE_INFO_NAME: ${{ matrix.arch }}-debian-11
CONBENCH_MACHINE_INFO_NAME: ${{ matrix.arch }}-debian-12
run: |
python3 -m pip install benchadapt@git+https://github.com/conbench/conbench.git@main#subdirectory=benchadapt/python
python3 ci/scripts/go_bench_adapt.py
Expand All@@ -181,7 +181,7 @@ jobs:
GOARCH=386 go build ./...

docker_cgo:
name: AMD64 Debian 11 Go ${{ matrix.go }} - CGO
name: AMD64 Debian 12 Go ${{ matrix.go }} - CGO
runs-on: ubuntu-latest
if: ${{ !contains(github.event.pull_request.title, 'WIP') }}
timeout-minutes: 20
Expand DownExpand Up@@ -222,7 +222,7 @@ jobs:


docker_cgo_python:
name: AMD64 Debian 11 Go ${{ matrix.go }} - CGO Python
name: AMD64 Debian 12 Go ${{ matrix.go }} - CGO Python
runs-on: ubuntu-latest
if: ${{ !contains(github.event.pull_request.title, 'WIP') }}
timeout-minutes: 15
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/js.yml
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -41,7 +41,7 @@ permissions:
jobs:

docker:
name: AMD64 Debian 11 NodeJS 18
name: AMD64 Debian 12 NodeJS 18
runs-on: ubuntu-latest
if: ${{ !contains(github.event.pull_request.title, 'WIP') }}
timeout-minutes: 60
Expand Down
64 changes: 39 additions & 25 deletions ci/docker/debian-11-cpp.dockerfile → ci/docker/debian-12-cpp.dockerfile
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -16,46 +16,42 @@
# under the License.

ARG arch=amd64
FROM ${arch}/debian:11
FROM ${arch}/debian:12
ARG arch

ENV DEBIAN_FRONTEND noninteractive

ARG llvm
# We can't use LLVM 14 or later from apt.llvm.org on i386 because LLVM
# 14 or later dropped support for i386.
RUN apt-get update -y -q && \
apt-get install -y -q --no-install-recommends \
dpkg-dev && \
latest_available_llvm_i386=13 && \
if [ $(dpkg-architecture -qDEB_HOST_ARCH) = "i386" -a \
"${llvm}" -gt "${latest_available_llvm_i386}" ]; then \
available_llvm="${latest_available_llvm_i386}"; \
else \
available_llvm="${llvm}"; \
fi && \
apt-get update -y -q && \
apt-get install -y -q --no-install-recommends \
apt-transport-https \
ca-certificates \
gnupg \
lsb-release \
wget && \
wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - && \
echo "deb https://apt.llvm.org/bullseye/ llvm-toolchain-bullseye-${available_llvm} main" > \
/etc/apt/sources.list.d/llvm.list && \
if [ ${llvm} -ge 17 ]; then \
wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | \
gpg \
--import - \
--keyring /usr/share/keyrings/llvm-snapshot.gpg \
--no-default-keyring && \
echo "deb[keyring=/usr/share/keyrings/llvm-snapshot.gpg] https://apt.llvm.org/$(lsb_release --codename --short)/ llvm-toolchain-$(lsb_release --codename --short)-${available_llvm} main" > \
/etc/apt/sources.list.d/llvm.list; \
fi && \
apt-get update -y -q && \
apt-get install -y -q --no-install-recommends \
autoconf \
ccache \
clang-${available_llvm} \
clang-${llvm} \
cmake \
curl \
g++ \
gcc \
gdb \
git \
libbenchmark-dev \
libboost-all-dev \
libboost-filesystem-dev \
libboost-system-dev \
libbrotli-dev \
libbz2-dev \
libc-ares-dev \
Expand All@@ -64,22 +60,35 @@ RUN apt-get update -y -q && \
libgmock-dev \
libgoogle-glog-dev \
libgrpc++-dev \
libidn2-dev \
libkrb5-dev \
libldap-dev \
liblz4-dev \
libnghttp2-dev \
libprotobuf-dev \
libprotoc-dev \
libpsl-dev \
libre2-dev \
librtmp-dev \
libsnappy-dev \
libsqlite3-dev \
libssh-dev \
libssh2-1-dev \
libssl-dev \
libthrift-dev \
libutf8proc-dev \
libxml2-dev \
libzstd-dev \
llvm-${available_llvm}-dev \
llvm-${llvm}-dev \
make \
ninja-build \
nlohmann-json3-dev \
npm \
pkg-config \
protobuf-compiler-grpc \
python3-dev \
python3-pip \
python3-venv \
rapidjson-dev \
rsync \
tzdata \
Expand All@@ -93,18 +102,24 @@ RUN /arrow/ci/scripts/install_minio.sh latest /usr/local
COPY ci/scripts/install_gcs_testbench.sh /arrow/ci/scripts/
RUN /arrow/ci/scripts/install_gcs_testbench.sh default

COPY ci/scripts/install_azurite.sh /arrow/ci/scripts/
RUN /arrow/ci/scripts/install_azurite.sh

COPY ci/scripts/install_sccache.sh /arrow/ci/scripts/
RUN /arrow/ci/scripts/install_sccache.sh unknown-linux-musl /usr/local/bin

ENV absl_SOURCE=BUNDLED \
ARROW_ACERO=ON \
ENV ARROW_ACERO=ON \
ARROW_AZURE=ON \
ARROW_BUILD_TESTS=ON \
ARROW_DATASET=ON \
ARROW_DEPENDENCY_SOURCE=SYSTEM \
ARROW_DATASET=ON \
ARROW_FLIGHT=ON \
ARROW_FLIGHT_SQL=ON \
ARROW_GANDIVA=ON \
ARROW_GCS=ON \
ARROW_HOME=/usr/local \
ARROW_NO_DEPRECATED_API=ON \
ARROW_ORC=ON \
ARROW_PARQUET=ON \
ARROW_S3=ON \
Expand All@@ -113,15 +128,14 @@ ENV absl_SOURCE=BUNDLED \
ARROW_WITH_BROTLI=ON \
ARROW_WITH_BZ2=ON \
ARROW_WITH_LZ4=ON \
ARROW_WITH_OPENTELEMETRY=OFF \
ARROW_WITH_OPENTELEMETRY=ON \
ARROW_WITH_SNAPPY=ON \
ARROW_WITH_ZLIB=ON \
ARROW_WITH_ZSTD=ON \
AWSSDK_SOURCE=BUNDLED \
CC=gcc \
CXX=g++ \
Azure_SOURCE=BUNDLED \
google_cloud_cpp_storage_SOURCE=BUNDLED \
GTest_SOURCE=BUNDLED \
ORC_SOURCE=BUNDLED \
PATH=/usr/lib/ccache/:$PATH \
PYTHON=python3 \
xsimd_SOURCE=BUNDLED
14 changes: 6 additions & 8 deletions ci/docker/debian-11-go-cgo-python.dockerfile → ci/docker/debian-12-go-cgo-python.dockerfile
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -24,13 +24,11 @@ ENV DEBIAN_FRONTEND noninteractive
RUN apt-get update -y -q && \
apt-get install -y -q --no-install-recommends \
python3 \
python3-pip && \
python3-pip \
python3-venv && \
apt-get clean

RUN ln -s /usr/bin/python3 /usr/local/bin/python && \
ln -s /usr/bin/pip3 /usr/local/bin/pip

# Need a newer pip than Debian's to install manylinux201x wheels
RUN pip install -U pip

RUN pip install pyarrow cffi --only-binary pyarrow
ENV ARROW_PYTHON_VENV /arrow-dev
RUN python3 -m venv ${ARROW_PYTHON_VENV} && \
. ${ARROW_PYTHON_VENV}/bin/activate && \
pip install pyarrow cffi --only-binary pyarrow
2 changes: 1 addition & 1 deletion ci/docker/debian-11-go.dockerfile → ci/docker/debian-12-go.dockerfile
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -18,7 +18,7 @@
ARG arch=amd64
ARG go=1.19
ARG staticcheck=v0.4.5
FROM ${arch}/golang:${go}-bullseye
FROM ${arch}/golang:${go}-bookworm

# FROM collects all the args, get back the staticcheck version arg
ARG staticcheck
Expand Down
2 changes: 1 addition & 1 deletion ci/docker/debian-11-js.dockerfile → ci/docker/debian-12-js.dockerfile
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -16,7 +16,7 @@
# under the License.

ARG arch=amd64
ARG node=16
ARG node=18
FROM ${arch}/node:${node}

ENV NODE_NO_WARNINGS=1
Expand Down
20 changes: 2 additions & 18 deletions ci/docker/linux-apt-c-glib.dockerfile
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -25,6 +25,7 @@ RUN apt-get update -y -q && \
libglib2.0-doc \
lsb-release \
luarocks \
meson \
ninja-build \
pkg-config \
python3 \
Expand All@@ -36,24 +37,7 @@ RUN apt-get update -y -q && \

RUN luarocks install lgi

# pip on Ubuntu 20.04 may be buggy:
#
# Collecting meson
# Downloading meson-0.53.2.tar.gz (1.6 MB)
# Installing build dependencies: started
# Installing build dependencies: finished with status 'done'
# Getting requirements to build wheel: started
# Getting requirements to build wheel: finished with status 'error'
# ERROR: Command errored out with exit status 1:
# command: /usr/bin/python3 /usr/share/python-wheels/pep517-0.7.0-py2.py3-none-any.whl/pep517/_in_process.py get_requires_for_build_wheel /tmp/tmpsk4jveay
# cwd: /tmp/pip-install-jn79a_kh/meson
# Complete output (1 lines):
# /usr/bin/python3: can't find '__main__' module in '/usr/share/python-wheels/pep517-0.7.0-py2.py3-none-any.whl/pep517/_in_process.py'
# ----------------------------------------
# ERROR: Command errored out with exit status 1: /usr/bin/python3 /usr/share/python-wheels/pep517-0.7.0-py2.py3-none-any.whl/pep517/_in_process.py get_requires_for_build_wheel /tmp/tmpsk4jveay Check the logs for full command output.
RUN (python3 -m pip install meson || \
python3 -m pip install --no-use-pep517 meson) && \
gem install --no-document bundler
RUN gem install --no-document bundler

COPY c_glib/Gemfile /arrow/c_glib/
RUN bundle install --gemfile /arrow/c_glib/Gemfile
Expand Down
23 changes: 7 additions & 16 deletions ci/docker/linux-apt-python-3.dockerfile
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -18,26 +18,17 @@
ARG base
FROM ${base}

RUN apt-get update -y -q && \
apt-get install -y -q \
python3 \
python3-pip \
python3-dev && \
apt-get clean && \
rm -rf /var/lib/apt/lists/*

RUN ln -s /usr/bin/python3 /usr/local/bin/python && \
ln -s /usr/bin/pip3 /usr/local/bin/pip

RUN pip install -U pip setuptools wheel

COPY python/requirements-build.txt \
python/requirements-test.txt \
/arrow/python/

RUN pip install \
-r arrow/python/requirements-build.txt \
-r arrow/python/requirements-test.txt
ENV ARROW_PYTHON_VENV /arrow-dev
RUN python3 -m venv ${ARROW_PYTHON_VENV} && \
. ${ARROW_PYTHON_VENV}/bin/activate && \
pip install -U pip setuptools wheel && \
pip install \
-r arrow/python/requirements-build.txt \
-r arrow/python/requirements-test.txt

ARG numba
COPY ci/scripts/install_numba.sh /arrow/ci/scripts/
Expand Down
4 changes: 4 additions & 0 deletions ci/scripts/go_cgo_python_test.sh
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -21,6 +21,10 @@ set -ex

source_dir=${1}/go

if [ -n "${ARROW_PYTHON_VENV:-}" ]; then
. "${ARROW_PYTHON_VENV}/bin/activate"
fi

export GOFLAGS="${GOFLAGS} -gcflags=all=-d=checkptr"

pushd ${source_dir}/arrow/cdata/test
Expand Down
4 changes: 4 additions & 0 deletions ci/scripts/install_numba.sh
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -26,6 +26,10 @@ fi

numba=$1

if [ -n "${ARROW_PYTHON_VENV:-}" ]; then
. "${ARROW_PYTHON_VENV}/bin/activate"
fi

if [ "${numba}" = "master" ]; then
pip install https://github.com/numba/numba/archive/main.tar.gz#egg=numba
elif [ "${numba}" = "latest" ]; then
Expand Down
4 changes: 4 additions & 0 deletions ci/scripts/python_build.sh
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -31,6 +31,10 @@ if [ -x "$(command -v git)" ]; then
git config --global --add safe.directory ${arrow_dir}
fi

if [ -n "${ARROW_PYTHON_VENV:-}" ]; then
. "${ARROW_PYTHON_VENV}/bin/activate"
fi

case "$(uname)" in
Linux)
n_jobs=$(nproc)
Expand Down
4 changes: 4 additions & 0 deletions ci/scripts/python_test.sh
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -22,6 +22,10 @@ set -ex
arrow_dir=${1}
test_dir=${1}/python/build/dist

if [ -n "${ARROW_PYTHON_VENV:-}" ]; then
. "${ARROW_PYTHON_VENV}/bin/activate"
fi

export ARROW_SOURCE_DIR=${arrow_dir}
export ARROW_TEST_DATA=${arrow_dir}/testing/data
export PARQUET_TEST_DATA=${arrow_dir}/cpp/submodules/parquet-testing/data
Expand Down
2 changes: 0 additions & 2 deletions dev/tasks/docker-tests/github.linux.yml
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -27,9 +27,7 @@ jobs:
{{ macros.github_set_env(env) }}
steps:
{{ macros.github_checkout_arrow(fetch_depth=fetch_depth|default(1))|indent }}
{% if image in ["ubuntu-r-only-r"] %}
{{ macros.github_free_space()|indent }}
{% endif %}
{{ macros.github_install_archery()|indent }}

- name: Execute Docker Build
Expand Down
Loading
, 'i'); if (__m === '*' || __re.test(location.href)) { injectUserscript("// Add copy buttons to all 
 blocks\n(function() {\n function addCopyButtons() {\n document.querySelectorAll('pre code').forEach(function(codeBlock) {\n if (codeBlock.parentElement.hasAttribute('data-copy-added')) return;\n codeBlock.parentElement.setAttribute('data-copy-added', 'true');\n \n var btn = document.createElement('button');\n btn.textContent = 'Copy';\n btn.style.cssText = 'position:absolute;top:4px;right:4px;padding:2px 8px;font-size:11px;background:#4ecdc4;border:none;border-radius:4px;color:#1a1a2e;cursor:pointer;opacity:0.7;transition:opacity 0.2s;';\n btn.onmouseover = function() { this.style.opacity = '1'; };\n btn.onmouseout = function() { this.style.opacity = '0.7'; };\n btn.onclick = function() {\n navigator.clipboard.writeText(codeBlock.textContent).then(function() {\n btn.textContent = 'Copied!';\n setTimeout(function() { btn.textContent = 'Copy'; }, 1500);\n });\n };\n codeBlock.parentElement.style.position = 'relative';\n codeBlock.parentElement.appendChild(btn);\n });\n }\n \n addCopyButtons();\n \n // Re-run on dynamic content\n var observer = new MutationObserver(addCopyButtons);\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Add Copy Buttons to Code Blocks");
}
} catch(__e) { console.warn('[Userscript:Add Copy Buttons to Code Blocks]', __e); }
})();
(function(){
try {
var __m = "github.com";
var __re = new RegExp('^' + "github\\.com" + '

Skip to content
GH-40454: [CI][Debian] Update Debian to 12 from 11#40455

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

raulcd
merged 4 commits into

from

Mar 13, 2024
 Merged


GH-40454: [CI][Debian] Update Debian to 12 from 11
#40455
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


4 commits


Select commit
Hold shift + click to select a range
281c437

GH-40454: [CI][Debian] Update Debian to 12 from 11


kou Mar 11, 2024
3a5b568

Update pip


kou Mar 11, 2024
429fcda

Free space


kou Mar 12, 2024
8039954

Add missing $


kou Mar 12, 2024

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


2 changes: 1 addition & 1 deletion
.env

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -48,7 +48,7 @@ ULIMIT_CORE=-1
# Default versions for platforms
ALMALINUX=8
ALPINE_LINUX=3.16
DEBIAN=11
DEBIAN=12
FEDORA=39
UBUNTU=20.04

Expand Down

8 changes: 4 additions & 4 deletions
.github/workflows/go.yml

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -90,7 +90,7 @@ jobs:
 echo "JSON" >> "$GITHUB_OUTPUT"

 docker:
 name: ${{ matrix.arch-label }} Debian 11 Go ${{ matrix.go }}
 name: ${{ matrix.arch-label }} Debian 12 Go ${{ matrix.go }}
 needs: docker-targets
 runs-on: ${{ matrix.runs-on }}
 if: ${{ !contains(github.event.pull_request.title, 'WIP') }}
Expand DownExpand Up@@ -154,7 +154,7 @@ jobs:
 CONBENCH_EMAIL: ${{ secrets.CONBENCH_EMAIL }}
 CONBENCH_PASSWORD: ${{ secrets.CONBENCH_PASS }}
 CONBENCH_REF: ${{ github.ref_name }}
 CONBENCH_MACHINE_INFO_NAME: ${{ matrix.arch }}-debian-11
 CONBENCH_MACHINE_INFO_NAME: ${{ matrix.arch }}-debian-12
 run: |
 python3 -m pip install benchadapt@git+https://github.com/conbench/conbench.git@main#subdirectory=benchadapt/python
 python3 ci/scripts/go_bench_adapt.py
Expand All@@ -181,7 +181,7 @@ jobs:
 GOARCH=386 go build ./...

 docker_cgo:
 name: AMD64 Debian 11 Go ${{ matrix.go }} - CGO
 name: AMD64 Debian 12 Go ${{ matrix.go }} - CGO
 runs-on: ubuntu-latest
 if: ${{ !contains(github.event.pull_request.title, 'WIP') }}
 timeout-minutes: 20
Expand DownExpand Up@@ -222,7 +222,7 @@ jobs:


 docker_cgo_python:
 name: AMD64 Debian 11 Go ${{ matrix.go }} - CGO Python
 name: AMD64 Debian 12 Go ${{ matrix.go }} - CGO Python
 runs-on: ubuntu-latest
 if: ${{ !contains(github.event.pull_request.title, 'WIP') }}
 timeout-minutes: 15
Expand Down

2 changes: 1 addition & 1 deletion
.github/workflows/js.yml

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -41,7 +41,7 @@ permissions:
jobs:

 docker:
 name: AMD64 Debian 11 NodeJS 18
 name: AMD64 Debian 12 NodeJS 18
 runs-on: ubuntu-latest
 if: ${{ !contains(github.event.pull_request.title, 'WIP') }}
 timeout-minutes: 60
Expand Down

64 changes: 39 additions & 25 deletions
ci/docker/debian-11-cpp.dockerfile → ci/docker/debian-12-cpp.dockerfile

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -16,46 +16,42 @@
# under the License.

ARG arch=amd64
FROM ${arch}/debian:11
FROM ${arch}/debian:12
ARG arch

ENV DEBIAN_FRONTEND noninteractive

ARG llvm
# We can't use LLVM 14 or later from apt.llvm.org on i386 because LLVM
# 14 or later dropped support for i386.
RUN apt-get update -y -q && \
 apt-get install -y -q --no-install-recommends \
 dpkg-dev && \
 latest_available_llvm_i386=13 && \
 if [ $(dpkg-architecture -qDEB_HOST_ARCH) = "i386" -a \
 "${llvm}" -gt "${latest_available_llvm_i386}" ]; then \
 available_llvm="${latest_available_llvm_i386}"; \
 else \
 available_llvm="${llvm}"; \
 fi && \
 apt-get update -y -q && \
 apt-get install -y -q --no-install-recommends \
 apt-transport-https \
 ca-certificates \
 gnupg \
 lsb-release \
 wget && \
 wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - && \
 echo "deb https://apt.llvm.org/bullseye/ llvm-toolchain-bullseye-${available_llvm} main" > \
 /etc/apt/sources.list.d/llvm.list && \
 if [ ${llvm} -ge 17 ]; then \
 wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | \
 gpg \
 --import - \
 --keyring /usr/share/keyrings/llvm-snapshot.gpg \
 --no-default-keyring && \
 echo "deb[keyring=/usr/share/keyrings/llvm-snapshot.gpg] https://apt.llvm.org/$(lsb_release --codename --short)/ llvm-toolchain-$(lsb_release --codename --short)-${available_llvm} main" > \
 /etc/apt/sources.list.d/llvm.list; \
 fi && \
 apt-get update -y -q && \
 apt-get install -y -q --no-install-recommends \
 autoconf \
 ccache \
 clang-${available_llvm} \
 clang-${llvm} \
 cmake \
 curl \
 g++ \
 gcc \
 gdb \
 git \
 libbenchmark-dev \
 libboost-all-dev \
 libboost-filesystem-dev \
 libboost-system-dev \
 libbrotli-dev \
 libbz2-dev \
 libc-ares-dev \
Expand All@@ -64,22 +60,35 @@ RUN apt-get update -y -q && \
 libgmock-dev \
 libgoogle-glog-dev \
 libgrpc++-dev \
 libidn2-dev \
 libkrb5-dev \
 libldap-dev \
 liblz4-dev \
 libnghttp2-dev \
 libprotobuf-dev \
 libprotoc-dev \
 libpsl-dev \
 libre2-dev \
 librtmp-dev \
 libsnappy-dev \
 libsqlite3-dev \
 libssh-dev \
 libssh2-1-dev \
 libssl-dev \
 libthrift-dev \
 libutf8proc-dev \
 libxml2-dev \
 libzstd-dev \
 llvm-${available_llvm}-dev \
 llvm-${llvm}-dev \
 make \
 ninja-build \
 nlohmann-json3-dev \
 npm \
 pkg-config \
 protobuf-compiler-grpc \
 python3-dev \
 python3-pip \
 python3-venv \
 rapidjson-dev \
 rsync \
 tzdata \
Expand All@@ -93,18 +102,24 @@ RUN /arrow/ci/scripts/install_minio.sh latest /usr/local
COPY ci/scripts/install_gcs_testbench.sh /arrow/ci/scripts/
RUN /arrow/ci/scripts/install_gcs_testbench.sh default

COPY ci/scripts/install_azurite.sh /arrow/ci/scripts/
RUN /arrow/ci/scripts/install_azurite.sh

COPY ci/scripts/install_sccache.sh /arrow/ci/scripts/
RUN /arrow/ci/scripts/install_sccache.sh unknown-linux-musl /usr/local/bin

ENV absl_SOURCE=BUNDLED \
ARROW_ACERO=ON \
ENV ARROW_ACERO=ON \
ARROW_AZURE=ON \
 ARROW_BUILD_TESTS=ON \
 ARROW_DATASET=ON \
 ARROW_DEPENDENCY_SOURCE=SYSTEM \
 ARROW_DATASET=ON \
 ARROW_FLIGHT=ON \
 ARROW_FLIGHT_SQL=ON \
 ARROW_GANDIVA=ON \
 ARROW_GCS=ON \
 ARROW_HOME=/usr/local \
 ARROW_NO_DEPRECATED_API=ON \
 ARROW_ORC=ON \
 ARROW_PARQUET=ON \
 ARROW_S3=ON \
Expand All@@ -113,15 +128,14 @@ ENV absl_SOURCE=BUNDLED \
 ARROW_WITH_BROTLI=ON \
 ARROW_WITH_BZ2=ON \
 ARROW_WITH_LZ4=ON \
 ARROW_WITH_OPENTELEMETRY=OFF \
 ARROW_WITH_OPENTELEMETRY=ON \
 ARROW_WITH_SNAPPY=ON \
 ARROW_WITH_ZLIB=ON \
 ARROW_WITH_ZSTD=ON \
 AWSSDK_SOURCE=BUNDLED \
 CC=gcc \
 CXX=g++ \
 Azure_SOURCE=BUNDLED \
 google_cloud_cpp_storage_SOURCE=BUNDLED \
 GTest_SOURCE=BUNDLED \
 ORC_SOURCE=BUNDLED \
 PATH=/usr/lib/ccache/:$PATH \
 PYTHON=python3 \
 xsimd_SOURCE=BUNDLED

14 changes: 6 additions & 8 deletions
ci/docker/debian-11-go-cgo-python.dockerfile → ci/docker/debian-12-go-cgo-python.dockerfile

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -24,13 +24,11 @@ ENV DEBIAN_FRONTEND noninteractive
RUN apt-get update -y -q && \
 apt-get install -y -q --no-install-recommends \
 python3 \
 python3-pip && \
 python3-pip \
 python3-venv && \
 apt-get clean

RUN ln -s /usr/bin/python3 /usr/local/bin/python && \
 ln -s /usr/bin/pip3 /usr/local/bin/pip

# Need a newer pip than Debian's to install manylinux201x wheels
RUN pip install -U pip

RUN pip install pyarrow cffi --only-binary pyarrow
ENV ARROW_PYTHON_VENV /arrow-dev
RUN python3 -m venv ${ARROW_PYTHON_VENV} && \
 . ${ARROW_PYTHON_VENV}/bin/activate && \
 pip install pyarrow cffi --only-binary pyarrow

2 changes: 1 addition & 1 deletion
ci/docker/debian-11-go.dockerfile → ci/docker/debian-12-go.dockerfile

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -18,7 +18,7 @@
ARG arch=amd64
ARG go=1.19
ARG staticcheck=v0.4.5
FROM ${arch}/golang:${go}-bullseye
FROM ${arch}/golang:${go}-bookworm

# FROM collects all the args, get back the staticcheck version arg
ARG staticcheck
Expand Down

2 changes: 1 addition & 1 deletion
ci/docker/debian-11-js.dockerfile → ci/docker/debian-12-js.dockerfile

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -16,7 +16,7 @@
# under the License.

ARG arch=amd64
ARG node=16
ARG node=18
FROM ${arch}/node:${node}

ENV NODE_NO_WARNINGS=1
Expand Down

20 changes: 2 additions & 18 deletions
ci/docker/linux-apt-c-glib.dockerfile

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -25,6 +25,7 @@ RUN apt-get update -y -q && \
 libglib2.0-doc \
 lsb-release \
 luarocks \
 meson \
 ninja-build \
 pkg-config \
 python3 \
Expand All@@ -36,24 +37,7 @@ RUN apt-get update -y -q && \

RUN luarocks install lgi

# pip on Ubuntu 20.04 may be buggy:
#
# Collecting meson
# Downloading meson-0.53.2.tar.gz (1.6 MB)
# Installing build dependencies: started
# Installing build dependencies: finished with status 'done'
# Getting requirements to build wheel: started
# Getting requirements to build wheel: finished with status 'error'
# ERROR: Command errored out with exit status 1:
# command: /usr/bin/python3 /usr/share/python-wheels/pep517-0.7.0-py2.py3-none-any.whl/pep517/_in_process.py get_requires_for_build_wheel /tmp/tmpsk4jveay
# cwd: /tmp/pip-install-jn79a_kh/meson
# Complete output (1 lines):
# /usr/bin/python3: can't find '__main__' module in '/usr/share/python-wheels/pep517-0.7.0-py2.py3-none-any.whl/pep517/_in_process.py'
# ----------------------------------------
# ERROR: Command errored out with exit status 1: /usr/bin/python3 /usr/share/python-wheels/pep517-0.7.0-py2.py3-none-any.whl/pep517/_in_process.py get_requires_for_build_wheel /tmp/tmpsk4jveay Check the logs for full command output.
RUN (python3 -m pip install meson || \
 python3 -m pip install --no-use-pep517 meson) && \
 gem install --no-document bundler
RUN gem install --no-document bundler

COPY c_glib/Gemfile /arrow/c_glib/
RUN bundle install --gemfile /arrow/c_glib/Gemfile
Expand Down

23 changes: 7 additions & 16 deletions
ci/docker/linux-apt-python-3.dockerfile

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -18,26 +18,17 @@
ARG base
FROM ${base}

RUN apt-get update -y -q && \
 apt-get install -y -q \
 python3 \
 python3-pip \
 python3-dev && \
 apt-get clean && \
 rm -rf /var/lib/apt/lists/*

RUN ln -s /usr/bin/python3 /usr/local/bin/python && \
 ln -s /usr/bin/pip3 /usr/local/bin/pip

RUN pip install -U pip setuptools wheel

COPY python/requirements-build.txt \
 python/requirements-test.txt \
 /arrow/python/

RUN pip install \
 -r arrow/python/requirements-build.txt \
 -r arrow/python/requirements-test.txt
ENV ARROW_PYTHON_VENV /arrow-dev
RUN python3 -m venv ${ARROW_PYTHON_VENV} && \
 . ${ARROW_PYTHON_VENV}/bin/activate && \
 pip install -U pip setuptools wheel && \
 pip install \
 -r arrow/python/requirements-build.txt \
 -r arrow/python/requirements-test.txt

ARG numba
COPY ci/scripts/install_numba.sh /arrow/ci/scripts/
Expand Down

4 changes: 4 additions & 0 deletions
ci/scripts/go_cgo_python_test.sh

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -21,6 +21,10 @@ set -ex

source_dir=${1}/go

if [ -n "${ARROW_PYTHON_VENV:-}" ]; then
 . "${ARROW_PYTHON_VENV}/bin/activate"
fi

export GOFLAGS="${GOFLAGS} -gcflags=all=-d=checkptr"

pushd ${source_dir}/arrow/cdata/test
Expand Down

4 changes: 4 additions & 0 deletions
ci/scripts/install_numba.sh

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -26,6 +26,10 @@ fi

numba=$1

if [ -n "${ARROW_PYTHON_VENV:-}" ]; then
 . "${ARROW_PYTHON_VENV}/bin/activate"
fi

if [ "${numba}" = "master" ]; then
 pip install https://github.com/numba/numba/archive/main.tar.gz#egg=numba
elif [ "${numba}" = "latest" ]; then
Expand Down

4 changes: 4 additions & 0 deletions
ci/scripts/python_build.sh

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -31,6 +31,10 @@ if [ -x "$(command -v git)" ]; then
 git config --global --add safe.directory ${arrow_dir}
fi

if [ -n "${ARROW_PYTHON_VENV:-}" ]; then
 . "${ARROW_PYTHON_VENV}/bin/activate"
fi

case "$(uname)" in
 Linux)
 n_jobs=$(nproc)
Expand Down

4 changes: 4 additions & 0 deletions
ci/scripts/python_test.sh

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -22,6 +22,10 @@ set -ex
arrow_dir=${1}
test_dir=${1}/python/build/dist

if [ -n "${ARROW_PYTHON_VENV:-}" ]; then
 . "${ARROW_PYTHON_VENV}/bin/activate"
fi

export ARROW_SOURCE_DIR=${arrow_dir}
export ARROW_TEST_DATA=${arrow_dir}/testing/data
export PARQUET_TEST_DATA=${arrow_dir}/cpp/submodules/parquet-testing/data
Expand Down

2 changes: 0 additions & 2 deletions
dev/tasks/docker-tests/github.linux.yml

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -27,9 +27,7 @@ jobs:
{{ macros.github_set_env(env) }}
 steps:
 {{ macros.github_checkout_arrow(fetch_depth=fetch_depth|default(1))|indent }}
 {% if image in ["ubuntu-r-only-r"] %}
 {{ macros.github_free_space()|indent }}
 {% endif %}
 {{ macros.github_install_archery()|indent }}

 - name: Execute Docker Build
Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Force GitHub README to respect dark mode\n(function() {\n var style = document.createElement('style');\n style.textContent = '\n .markdown-body {\n color-scheme: dark light;\n }\n .markdown-body pre { background: #161b22 !important; }\n .markdown-body code { background: rgba(110, 118, 129, 0.4) !important; }\n .markdown-body table th, .markdown-body table td { border-color: #30363d !important; }\n .markdown-body img { background: #0d1117; }\n .markdown-body blockquote { border-left-color: #8b949e; }\n .markdown-body hr { border-color: #30363d; }\n ';\n document.head.appendChild(style);\n})();", "GitHub Dark Mode README Fix");
}
} catch(__e) { console.warn('[Userscript:GitHub Dark Mode README Fix]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
GH-40454: [CI][Debian] Update Debian to 12 from 11#40455

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

raulcd
merged 4 commits into

from

Mar 13, 2024
 Merged


GH-40454: [CI][Debian] Update Debian to 12 from 11
#40455
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


4 commits


Select commit
Hold shift + click to select a range
281c437

GH-40454: [CI][Debian] Update Debian to 12 from 11


kou Mar 11, 2024
3a5b568

Update pip


kou Mar 11, 2024
429fcda

Free space


kou Mar 12, 2024
8039954

Add missing $


kou Mar 12, 2024

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


2 changes: 1 addition & 1 deletion
.env

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -48,7 +48,7 @@ ULIMIT_CORE=-1
# Default versions for platforms
ALMALINUX=8
ALPINE_LINUX=3.16
DEBIAN=11
DEBIAN=12
FEDORA=39
UBUNTU=20.04

Expand Down

8 changes: 4 additions & 4 deletions
.github/workflows/go.yml

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -90,7 +90,7 @@ jobs:
 echo "JSON" >> "$GITHUB_OUTPUT"

 docker:
 name: ${{ matrix.arch-label }} Debian 11 Go ${{ matrix.go }}
 name: ${{ matrix.arch-label }} Debian 12 Go ${{ matrix.go }}
 needs: docker-targets
 runs-on: ${{ matrix.runs-on }}
 if: ${{ !contains(github.event.pull_request.title, 'WIP') }}
Expand DownExpand Up@@ -154,7 +154,7 @@ jobs:
 CONBENCH_EMAIL: ${{ secrets.CONBENCH_EMAIL }}
 CONBENCH_PASSWORD: ${{ secrets.CONBENCH_PASS }}
 CONBENCH_REF: ${{ github.ref_name }}
 CONBENCH_MACHINE_INFO_NAME: ${{ matrix.arch }}-debian-11
 CONBENCH_MACHINE_INFO_NAME: ${{ matrix.arch }}-debian-12
 run: |
 python3 -m pip install benchadapt@git+https://github.com/conbench/conbench.git@main#subdirectory=benchadapt/python
 python3 ci/scripts/go_bench_adapt.py
Expand All@@ -181,7 +181,7 @@ jobs:
 GOARCH=386 go build ./...

 docker_cgo:
 name: AMD64 Debian 11 Go ${{ matrix.go }} - CGO
 name: AMD64 Debian 12 Go ${{ matrix.go }} - CGO
 runs-on: ubuntu-latest
 if: ${{ !contains(github.event.pull_request.title, 'WIP') }}
 timeout-minutes: 20
Expand DownExpand Up@@ -222,7 +222,7 @@ jobs:


 docker_cgo_python:
 name: AMD64 Debian 11 Go ${{ matrix.go }} - CGO Python
 name: AMD64 Debian 12 Go ${{ matrix.go }} - CGO Python
 runs-on: ubuntu-latest
 if: ${{ !contains(github.event.pull_request.title, 'WIP') }}
 timeout-minutes: 15
Expand Down

2 changes: 1 addition & 1 deletion
.github/workflows/js.yml

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -41,7 +41,7 @@ permissions:
jobs:

 docker:
 name: AMD64 Debian 11 NodeJS 18
 name: AMD64 Debian 12 NodeJS 18
 runs-on: ubuntu-latest
 if: ${{ !contains(github.event.pull_request.title, 'WIP') }}
 timeout-minutes: 60
Expand Down

64 changes: 39 additions & 25 deletions
ci/docker/debian-11-cpp.dockerfile → ci/docker/debian-12-cpp.dockerfile

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -16,46 +16,42 @@
# under the License.

ARG arch=amd64
FROM ${arch}/debian:11
FROM ${arch}/debian:12
ARG arch

ENV DEBIAN_FRONTEND noninteractive

ARG llvm
# We can't use LLVM 14 or later from apt.llvm.org on i386 because LLVM
# 14 or later dropped support for i386.
RUN apt-get update -y -q && \
 apt-get install -y -q --no-install-recommends \
 dpkg-dev && \
 latest_available_llvm_i386=13 && \
 if [ $(dpkg-architecture -qDEB_HOST_ARCH) = "i386" -a \
 "${llvm}" -gt "${latest_available_llvm_i386}" ]; then \
 available_llvm="${latest_available_llvm_i386}"; \
 else \
 available_llvm="${llvm}"; \
 fi && \
 apt-get update -y -q && \
 apt-get install -y -q --no-install-recommends \
 apt-transport-https \
 ca-certificates \
 gnupg \
 lsb-release \
 wget && \
 wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - && \
 echo "deb https://apt.llvm.org/bullseye/ llvm-toolchain-bullseye-${available_llvm} main" > \
 /etc/apt/sources.list.d/llvm.list && \
 if [ ${llvm} -ge 17 ]; then \
 wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | \
 gpg \
 --import - \
 --keyring /usr/share/keyrings/llvm-snapshot.gpg \
 --no-default-keyring && \
 echo "deb[keyring=/usr/share/keyrings/llvm-snapshot.gpg] https://apt.llvm.org/$(lsb_release --codename --short)/ llvm-toolchain-$(lsb_release --codename --short)-${available_llvm} main" > \
 /etc/apt/sources.list.d/llvm.list; \
 fi && \
 apt-get update -y -q && \
 apt-get install -y -q --no-install-recommends \
 autoconf \
 ccache \
 clang-${available_llvm} \
 clang-${llvm} \
 cmake \
 curl \
 g++ \
 gcc \
 gdb \
 git \
 libbenchmark-dev \
 libboost-all-dev \
 libboost-filesystem-dev \
 libboost-system-dev \
 libbrotli-dev \
 libbz2-dev \
 libc-ares-dev \
Expand All@@ -64,22 +60,35 @@ RUN apt-get update -y -q && \
 libgmock-dev \
 libgoogle-glog-dev \
 libgrpc++-dev \
 libidn2-dev \
 libkrb5-dev \
 libldap-dev \
 liblz4-dev \
 libnghttp2-dev \
 libprotobuf-dev \
 libprotoc-dev \
 libpsl-dev \
 libre2-dev \
 librtmp-dev \
 libsnappy-dev \
 libsqlite3-dev \
 libssh-dev \
 libssh2-1-dev \
 libssl-dev \
 libthrift-dev \
 libutf8proc-dev \
 libxml2-dev \
 libzstd-dev \
 llvm-${available_llvm}-dev \
 llvm-${llvm}-dev \
 make \
 ninja-build \
 nlohmann-json3-dev \
 npm \
 pkg-config \
 protobuf-compiler-grpc \
 python3-dev \
 python3-pip \
 python3-venv \
 rapidjson-dev \
 rsync \
 tzdata \
Expand All@@ -93,18 +102,24 @@ RUN /arrow/ci/scripts/install_minio.sh latest /usr/local
COPY ci/scripts/install_gcs_testbench.sh /arrow/ci/scripts/
RUN /arrow/ci/scripts/install_gcs_testbench.sh default

COPY ci/scripts/install_azurite.sh /arrow/ci/scripts/
RUN /arrow/ci/scripts/install_azurite.sh

COPY ci/scripts/install_sccache.sh /arrow/ci/scripts/
RUN /arrow/ci/scripts/install_sccache.sh unknown-linux-musl /usr/local/bin

ENV absl_SOURCE=BUNDLED \
ARROW_ACERO=ON \
ENV ARROW_ACERO=ON \
ARROW_AZURE=ON \
 ARROW_BUILD_TESTS=ON \
 ARROW_DATASET=ON \
 ARROW_DEPENDENCY_SOURCE=SYSTEM \
 ARROW_DATASET=ON \
 ARROW_FLIGHT=ON \
 ARROW_FLIGHT_SQL=ON \
 ARROW_GANDIVA=ON \
 ARROW_GCS=ON \
 ARROW_HOME=/usr/local \
 ARROW_NO_DEPRECATED_API=ON \
 ARROW_ORC=ON \
 ARROW_PARQUET=ON \
 ARROW_S3=ON \
Expand All@@ -113,15 +128,14 @@ ENV absl_SOURCE=BUNDLED \
 ARROW_WITH_BROTLI=ON \
 ARROW_WITH_BZ2=ON \
 ARROW_WITH_LZ4=ON \
 ARROW_WITH_OPENTELEMETRY=OFF \
 ARROW_WITH_OPENTELEMETRY=ON \
 ARROW_WITH_SNAPPY=ON \
 ARROW_WITH_ZLIB=ON \
 ARROW_WITH_ZSTD=ON \
 AWSSDK_SOURCE=BUNDLED \
 CC=gcc \
 CXX=g++ \
 Azure_SOURCE=BUNDLED \
 google_cloud_cpp_storage_SOURCE=BUNDLED \
 GTest_SOURCE=BUNDLED \
 ORC_SOURCE=BUNDLED \
 PATH=/usr/lib/ccache/:$PATH \
 PYTHON=python3 \
 xsimd_SOURCE=BUNDLED

14 changes: 6 additions & 8 deletions
ci/docker/debian-11-go-cgo-python.dockerfile → ci/docker/debian-12-go-cgo-python.dockerfile

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -24,13 +24,11 @@ ENV DEBIAN_FRONTEND noninteractive
RUN apt-get update -y -q && \
 apt-get install -y -q --no-install-recommends \
 python3 \
 python3-pip && \
 python3-pip \
 python3-venv && \
 apt-get clean

RUN ln -s /usr/bin/python3 /usr/local/bin/python && \
 ln -s /usr/bin/pip3 /usr/local/bin/pip

# Need a newer pip than Debian's to install manylinux201x wheels
RUN pip install -U pip

RUN pip install pyarrow cffi --only-binary pyarrow
ENV ARROW_PYTHON_VENV /arrow-dev
RUN python3 -m venv ${ARROW_PYTHON_VENV} && \
 . ${ARROW_PYTHON_VENV}/bin/activate && \
 pip install pyarrow cffi --only-binary pyarrow

2 changes: 1 addition & 1 deletion
ci/docker/debian-11-go.dockerfile → ci/docker/debian-12-go.dockerfile

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -18,7 +18,7 @@
ARG arch=amd64
ARG go=1.19
ARG staticcheck=v0.4.5
FROM ${arch}/golang:${go}-bullseye
FROM ${arch}/golang:${go}-bookworm

# FROM collects all the args, get back the staticcheck version arg
ARG staticcheck
Expand Down

2 changes: 1 addition & 1 deletion
ci/docker/debian-11-js.dockerfile → ci/docker/debian-12-js.dockerfile

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -16,7 +16,7 @@
# under the License.

ARG arch=amd64
ARG node=16
ARG node=18
FROM ${arch}/node:${node}

ENV NODE_NO_WARNINGS=1
Expand Down

20 changes: 2 additions & 18 deletions
ci/docker/linux-apt-c-glib.dockerfile

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -25,6 +25,7 @@ RUN apt-get update -y -q && \
 libglib2.0-doc \
 lsb-release \
 luarocks \
 meson \
 ninja-build \
 pkg-config \
 python3 \
Expand All@@ -36,24 +37,7 @@ RUN apt-get update -y -q && \

RUN luarocks install lgi

# pip on Ubuntu 20.04 may be buggy:
#
# Collecting meson
# Downloading meson-0.53.2.tar.gz (1.6 MB)
# Installing build dependencies: started
# Installing build dependencies: finished with status 'done'
# Getting requirements to build wheel: started
# Getting requirements to build wheel: finished with status 'error'
# ERROR: Command errored out with exit status 1:
# command: /usr/bin/python3 /usr/share/python-wheels/pep517-0.7.0-py2.py3-none-any.whl/pep517/_in_process.py get_requires_for_build_wheel /tmp/tmpsk4jveay
# cwd: /tmp/pip-install-jn79a_kh/meson
# Complete output (1 lines):
# /usr/bin/python3: can't find '__main__' module in '/usr/share/python-wheels/pep517-0.7.0-py2.py3-none-any.whl/pep517/_in_process.py'
# ----------------------------------------
# ERROR: Command errored out with exit status 1: /usr/bin/python3 /usr/share/python-wheels/pep517-0.7.0-py2.py3-none-any.whl/pep517/_in_process.py get_requires_for_build_wheel /tmp/tmpsk4jveay Check the logs for full command output.
RUN (python3 -m pip install meson || \
 python3 -m pip install --no-use-pep517 meson) && \
 gem install --no-document bundler
RUN gem install --no-document bundler

COPY c_glib/Gemfile /arrow/c_glib/
RUN bundle install --gemfile /arrow/c_glib/Gemfile
Expand Down

23 changes: 7 additions & 16 deletions
ci/docker/linux-apt-python-3.dockerfile

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -18,26 +18,17 @@
ARG base
FROM ${base}

RUN apt-get update -y -q && \
 apt-get install -y -q \
 python3 \
 python3-pip \
 python3-dev && \
 apt-get clean && \
 rm -rf /var/lib/apt/lists/*

RUN ln -s /usr/bin/python3 /usr/local/bin/python && \
 ln -s /usr/bin/pip3 /usr/local/bin/pip

RUN pip install -U pip setuptools wheel

COPY python/requirements-build.txt \
 python/requirements-test.txt \
 /arrow/python/

RUN pip install \
 -r arrow/python/requirements-build.txt \
 -r arrow/python/requirements-test.txt
ENV ARROW_PYTHON_VENV /arrow-dev
RUN python3 -m venv ${ARROW_PYTHON_VENV} && \
 . ${ARROW_PYTHON_VENV}/bin/activate && \
 pip install -U pip setuptools wheel && \
 pip install \
 -r arrow/python/requirements-build.txt \
 -r arrow/python/requirements-test.txt

ARG numba
COPY ci/scripts/install_numba.sh /arrow/ci/scripts/
Expand Down

4 changes: 4 additions & 0 deletions
ci/scripts/go_cgo_python_test.sh

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -21,6 +21,10 @@ set -ex

source_dir=${1}/go

if [ -n "${ARROW_PYTHON_VENV:-}" ]; then
 . "${ARROW_PYTHON_VENV}/bin/activate"
fi

export GOFLAGS="${GOFLAGS} -gcflags=all=-d=checkptr"

pushd ${source_dir}/arrow/cdata/test
Expand Down

4 changes: 4 additions & 0 deletions
ci/scripts/install_numba.sh

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -26,6 +26,10 @@ fi

numba=$1

if [ -n "${ARROW_PYTHON_VENV:-}" ]; then
 . "${ARROW_PYTHON_VENV}/bin/activate"
fi

if [ "${numba}" = "master" ]; then
 pip install https://github.com/numba/numba/archive/main.tar.gz#egg=numba
elif [ "${numba}" = "latest" ]; then
Expand Down

4 changes: 4 additions & 0 deletions
ci/scripts/python_build.sh

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -31,6 +31,10 @@ if [ -x "$(command -v git)" ]; then
 git config --global --add safe.directory ${arrow_dir}
fi

if [ -n "${ARROW_PYTHON_VENV:-}" ]; then
 . "${ARROW_PYTHON_VENV}/bin/activate"
fi

case "$(uname)" in
 Linux)
 n_jobs=$(nproc)
Expand Down

4 changes: 4 additions & 0 deletions
ci/scripts/python_test.sh

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -22,6 +22,10 @@ set -ex
arrow_dir=${1}
test_dir=${1}/python/build/dist

if [ -n "${ARROW_PYTHON_VENV:-}" ]; then
 . "${ARROW_PYTHON_VENV}/bin/activate"
fi

export ARROW_SOURCE_DIR=${arrow_dir}
export ARROW_TEST_DATA=${arrow_dir}/testing/data
export PARQUET_TEST_DATA=${arrow_dir}/cpp/submodules/parquet-testing/data
Expand Down

2 changes: 0 additions & 2 deletions
dev/tasks/docker-tests/github.linux.yml

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -27,9 +27,7 @@ jobs:
{{ macros.github_set_env(env) }}
 steps:
 {{ macros.github_checkout_arrow(fetch_depth=fetch_depth|default(1))|indent }}
 {% if image in ["ubuntu-r-only-r"] %}
 {{ macros.github_free_space()|indent }}
 {% endif %}
 {{ macros.github_install_archery()|indent }}

 - name: Execute Docker Build
Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Highlight search terms from Google/DuckDuckGo/Bing referrer\n(function() {\n var ref = document.referrer;\n var terms = [];\n \n if (ref.includes('google.com') || ref.includes('duckduckgo.com') || ref.includes('bing.com')) {\n var url = new URL(ref);\n var q = url.searchParams.get('q') || url.searchParams.get('p');\n if (q) {\n terms = q.split(/\\s+/).filter(function(t) { return t.length > 2; });\n }\n }\n \n if (terms.length === 0) return;\n \n var style = document.createElement('style');\n style.textContent = '.userscript-highlight { background: #fbbf24; color: #1a1a2e; padding: 1px 3px; border-radius: 2px; }';\n document.head.appendChild(style);\n \n function highlight(node) {\n if (node.nodeType === 3) { // text node\n var text = node.textContent;\n var found = false;\n terms.forEach(function(term) {\n var regex = new RegExp('(' + term.replace(/[.*+?^${}()|[\\]\\\\]/g, '\\\\') + ')', 'gi');\n if (regex.test(text)) {\n found = true;\n var frag = document.createDocumentFragment();\n var parts = text.split(regex);\n parts.forEach(function(part, i) {\n if (i % 2 === 0) {\n frag.appendChild(document.createTextNode(part));\n } else {\n var span = document.createElement('span');\n span.className = 'userscript-highlight';\n span.textContent = part;\n frag.appendChild(span);\n }\n });\n node.parentNode.replaceChild(frag, node);\n }\n });\n } else if (node.nodeType === 1 && node.childNodes) { // element\n var skipTags = ['SCRIPT', 'STYLE', 'NOSCRIPT', 'TEXTAREA', 'INPUT', 'SELECT'];\n if (!skipTags.includes(node.tagName)) {\n Array.from(node.childNodes).forEach(highlight);\n }\n }\n }\n \n highlight(document.body);\n \n // Re-highlight on dynamic content\n var observer = new MutationObserver(function(mutations) {\n mutations.forEach(function(m) {\n m.addedNodes.forEach(function(node) {\n if (node.nodeType === 1 || node.nodeType === 3) highlight(node);\n });\n });\n });\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Highlight Search Terms");
}
} catch(__e) { console.warn('[Userscript:Highlight Search Terms]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
GH-40454: [CI][Debian] Update Debian to 12 from 11#40455

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

raulcd
merged 4 commits into

from

Mar 13, 2024
 Merged


GH-40454: [CI][Debian] Update Debian to 12 from 11
#40455
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


4 commits


Select commit
Hold shift + click to select a range
281c437

GH-40454: [CI][Debian] Update Debian to 12 from 11


kou Mar 11, 2024
3a5b568

Update pip


kou Mar 11, 2024
429fcda

Free space


kou Mar 12, 2024
8039954

Add missing $


kou Mar 12, 2024

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


2 changes: 1 addition & 1 deletion
.env

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -48,7 +48,7 @@ ULIMIT_CORE=-1
# Default versions for platforms
ALMALINUX=8
ALPINE_LINUX=3.16
DEBIAN=11
DEBIAN=12
FEDORA=39
UBUNTU=20.04

Expand Down

8 changes: 4 additions & 4 deletions
.github/workflows/go.yml

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -90,7 +90,7 @@ jobs:
 echo "JSON" >> "$GITHUB_OUTPUT"

 docker:
 name: ${{ matrix.arch-label }} Debian 11 Go ${{ matrix.go }}
 name: ${{ matrix.arch-label }} Debian 12 Go ${{ matrix.go }}
 needs: docker-targets
 runs-on: ${{ matrix.runs-on }}
 if: ${{ !contains(github.event.pull_request.title, 'WIP') }}
Expand DownExpand Up@@ -154,7 +154,7 @@ jobs:
 CONBENCH_EMAIL: ${{ secrets.CONBENCH_EMAIL }}
 CONBENCH_PASSWORD: ${{ secrets.CONBENCH_PASS }}
 CONBENCH_REF: ${{ github.ref_name }}
 CONBENCH_MACHINE_INFO_NAME: ${{ matrix.arch }}-debian-11
 CONBENCH_MACHINE_INFO_NAME: ${{ matrix.arch }}-debian-12
 run: |
 python3 -m pip install benchadapt@git+https://github.com/conbench/conbench.git@main#subdirectory=benchadapt/python
 python3 ci/scripts/go_bench_adapt.py
Expand All@@ -181,7 +181,7 @@ jobs:
 GOARCH=386 go build ./...

 docker_cgo:
 name: AMD64 Debian 11 Go ${{ matrix.go }} - CGO
 name: AMD64 Debian 12 Go ${{ matrix.go }} - CGO
 runs-on: ubuntu-latest
 if: ${{ !contains(github.event.pull_request.title, 'WIP') }}
 timeout-minutes: 20
Expand DownExpand Up@@ -222,7 +222,7 @@ jobs:


 docker_cgo_python:
 name: AMD64 Debian 11 Go ${{ matrix.go }} - CGO Python
 name: AMD64 Debian 12 Go ${{ matrix.go }} - CGO Python
 runs-on: ubuntu-latest
 if: ${{ !contains(github.event.pull_request.title, 'WIP') }}
 timeout-minutes: 15
Expand Down

2 changes: 1 addition & 1 deletion
.github/workflows/js.yml

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -41,7 +41,7 @@ permissions:
jobs:

 docker:
 name: AMD64 Debian 11 NodeJS 18
 name: AMD64 Debian 12 NodeJS 18
 runs-on: ubuntu-latest
 if: ${{ !contains(github.event.pull_request.title, 'WIP') }}
 timeout-minutes: 60
Expand Down

64 changes: 39 additions & 25 deletions
ci/docker/debian-11-cpp.dockerfile → ci/docker/debian-12-cpp.dockerfile

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -16,46 +16,42 @@
# under the License.

ARG arch=amd64
FROM ${arch}/debian:11
FROM ${arch}/debian:12
ARG arch

ENV DEBIAN_FRONTEND noninteractive

ARG llvm
# We can't use LLVM 14 or later from apt.llvm.org on i386 because LLVM
# 14 or later dropped support for i386.
RUN apt-get update -y -q && \
 apt-get install -y -q --no-install-recommends \
 dpkg-dev && \
 latest_available_llvm_i386=13 && \
 if [ $(dpkg-architecture -qDEB_HOST_ARCH) = "i386" -a \
 "${llvm}" -gt "${latest_available_llvm_i386}" ]; then \
 available_llvm="${latest_available_llvm_i386}"; \
 else \
 available_llvm="${llvm}"; \
 fi && \
 apt-get update -y -q && \
 apt-get install -y -q --no-install-recommends \
 apt-transport-https \
 ca-certificates \
 gnupg \
 lsb-release \
 wget && \
 wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - && \
 echo "deb https://apt.llvm.org/bullseye/ llvm-toolchain-bullseye-${available_llvm} main" > \
 /etc/apt/sources.list.d/llvm.list && \
 if [ ${llvm} -ge 17 ]; then \
 wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | \
 gpg \
 --import - \
 --keyring /usr/share/keyrings/llvm-snapshot.gpg \
 --no-default-keyring && \
 echo "deb[keyring=/usr/share/keyrings/llvm-snapshot.gpg] https://apt.llvm.org/$(lsb_release --codename --short)/ llvm-toolchain-$(lsb_release --codename --short)-${available_llvm} main" > \
 /etc/apt/sources.list.d/llvm.list; \
 fi && \
 apt-get update -y -q && \
 apt-get install -y -q --no-install-recommends \
 autoconf \
 ccache \
 clang-${available_llvm} \
 clang-${llvm} \
 cmake \
 curl \
 g++ \
 gcc \
 gdb \
 git \
 libbenchmark-dev \
 libboost-all-dev \
 libboost-filesystem-dev \
 libboost-system-dev \
 libbrotli-dev \
 libbz2-dev \
 libc-ares-dev \
Expand All@@ -64,22 +60,35 @@ RUN apt-get update -y -q && \
 libgmock-dev \
 libgoogle-glog-dev \
 libgrpc++-dev \
 libidn2-dev \
 libkrb5-dev \
 libldap-dev \
 liblz4-dev \
 libnghttp2-dev \
 libprotobuf-dev \
 libprotoc-dev \
 libpsl-dev \
 libre2-dev \
 librtmp-dev \
 libsnappy-dev \
 libsqlite3-dev \
 libssh-dev \
 libssh2-1-dev \
 libssl-dev \
 libthrift-dev \
 libutf8proc-dev \
 libxml2-dev \
 libzstd-dev \
 llvm-${available_llvm}-dev \
 llvm-${llvm}-dev \
 make \
 ninja-build \
 nlohmann-json3-dev \
 npm \
 pkg-config \
 protobuf-compiler-grpc \
 python3-dev \
 python3-pip \
 python3-venv \
 rapidjson-dev \
 rsync \
 tzdata \
Expand All@@ -93,18 +102,24 @@ RUN /arrow/ci/scripts/install_minio.sh latest /usr/local
COPY ci/scripts/install_gcs_testbench.sh /arrow/ci/scripts/
RUN /arrow/ci/scripts/install_gcs_testbench.sh default

COPY ci/scripts/install_azurite.sh /arrow/ci/scripts/
RUN /arrow/ci/scripts/install_azurite.sh

COPY ci/scripts/install_sccache.sh /arrow/ci/scripts/
RUN /arrow/ci/scripts/install_sccache.sh unknown-linux-musl /usr/local/bin

ENV absl_SOURCE=BUNDLED \
ARROW_ACERO=ON \
ENV ARROW_ACERO=ON \
ARROW_AZURE=ON \
 ARROW_BUILD_TESTS=ON \
 ARROW_DATASET=ON \
 ARROW_DEPENDENCY_SOURCE=SYSTEM \
 ARROW_DATASET=ON \
 ARROW_FLIGHT=ON \
 ARROW_FLIGHT_SQL=ON \
 ARROW_GANDIVA=ON \
 ARROW_GCS=ON \
 ARROW_HOME=/usr/local \
 ARROW_NO_DEPRECATED_API=ON \
 ARROW_ORC=ON \
 ARROW_PARQUET=ON \
 ARROW_S3=ON \
Expand All@@ -113,15 +128,14 @@ ENV absl_SOURCE=BUNDLED \
 ARROW_WITH_BROTLI=ON \
 ARROW_WITH_BZ2=ON \
 ARROW_WITH_LZ4=ON \
 ARROW_WITH_OPENTELEMETRY=OFF \
 ARROW_WITH_OPENTELEMETRY=ON \
 ARROW_WITH_SNAPPY=ON \
 ARROW_WITH_ZLIB=ON \
 ARROW_WITH_ZSTD=ON \
 AWSSDK_SOURCE=BUNDLED \
 CC=gcc \
 CXX=g++ \
 Azure_SOURCE=BUNDLED \
 google_cloud_cpp_storage_SOURCE=BUNDLED \
 GTest_SOURCE=BUNDLED \
 ORC_SOURCE=BUNDLED \
 PATH=/usr/lib/ccache/:$PATH \
 PYTHON=python3 \
 xsimd_SOURCE=BUNDLED

14 changes: 6 additions & 8 deletions
ci/docker/debian-11-go-cgo-python.dockerfile → ci/docker/debian-12-go-cgo-python.dockerfile

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -24,13 +24,11 @@ ENV DEBIAN_FRONTEND noninteractive
RUN apt-get update -y -q && \
 apt-get install -y -q --no-install-recommends \
 python3 \
 python3-pip && \
 python3-pip \
 python3-venv && \
 apt-get clean

RUN ln -s /usr/bin/python3 /usr/local/bin/python && \
 ln -s /usr/bin/pip3 /usr/local/bin/pip

# Need a newer pip than Debian's to install manylinux201x wheels
RUN pip install -U pip

RUN pip install pyarrow cffi --only-binary pyarrow
ENV ARROW_PYTHON_VENV /arrow-dev
RUN python3 -m venv ${ARROW_PYTHON_VENV} && \
 . ${ARROW_PYTHON_VENV}/bin/activate && \
 pip install pyarrow cffi --only-binary pyarrow

2 changes: 1 addition & 1 deletion
ci/docker/debian-11-go.dockerfile → ci/docker/debian-12-go.dockerfile

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -18,7 +18,7 @@
ARG arch=amd64
ARG go=1.19
ARG staticcheck=v0.4.5
FROM ${arch}/golang:${go}-bullseye
FROM ${arch}/golang:${go}-bookworm

# FROM collects all the args, get back the staticcheck version arg
ARG staticcheck
Expand Down

2 changes: 1 addition & 1 deletion
ci/docker/debian-11-js.dockerfile → ci/docker/debian-12-js.dockerfile

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -16,7 +16,7 @@
# under the License.

ARG arch=amd64
ARG node=16
ARG node=18
FROM ${arch}/node:${node}

ENV NODE_NO_WARNINGS=1
Expand Down

20 changes: 2 additions & 18 deletions
ci/docker/linux-apt-c-glib.dockerfile

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -25,6 +25,7 @@ RUN apt-get update -y -q && \
 libglib2.0-doc \
 lsb-release \
 luarocks \
 meson \
 ninja-build \
 pkg-config \
 python3 \
Expand All@@ -36,24 +37,7 @@ RUN apt-get update -y -q && \

RUN luarocks install lgi

# pip on Ubuntu 20.04 may be buggy:
#
# Collecting meson
# Downloading meson-0.53.2.tar.gz (1.6 MB)
# Installing build dependencies: started
# Installing build dependencies: finished with status 'done'
# Getting requirements to build wheel: started
# Getting requirements to build wheel: finished with status 'error'
# ERROR: Command errored out with exit status 1:
# command: /usr/bin/python3 /usr/share/python-wheels/pep517-0.7.0-py2.py3-none-any.whl/pep517/_in_process.py get_requires_for_build_wheel /tmp/tmpsk4jveay
# cwd: /tmp/pip-install-jn79a_kh/meson
# Complete output (1 lines):
# /usr/bin/python3: can't find '__main__' module in '/usr/share/python-wheels/pep517-0.7.0-py2.py3-none-any.whl/pep517/_in_process.py'
# ----------------------------------------
# ERROR: Command errored out with exit status 1: /usr/bin/python3 /usr/share/python-wheels/pep517-0.7.0-py2.py3-none-any.whl/pep517/_in_process.py get_requires_for_build_wheel /tmp/tmpsk4jveay Check the logs for full command output.
RUN (python3 -m pip install meson || \
 python3 -m pip install --no-use-pep517 meson) && \
 gem install --no-document bundler
RUN gem install --no-document bundler

COPY c_glib/Gemfile /arrow/c_glib/
RUN bundle install --gemfile /arrow/c_glib/Gemfile
Expand Down

23 changes: 7 additions & 16 deletions
ci/docker/linux-apt-python-3.dockerfile

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -18,26 +18,17 @@
ARG base
FROM ${base}

RUN apt-get update -y -q && \
 apt-get install -y -q \
 python3 \
 python3-pip \
 python3-dev && \
 apt-get clean && \
 rm -rf /var/lib/apt/lists/*

RUN ln -s /usr/bin/python3 /usr/local/bin/python && \
 ln -s /usr/bin/pip3 /usr/local/bin/pip

RUN pip install -U pip setuptools wheel

COPY python/requirements-build.txt \
 python/requirements-test.txt \
 /arrow/python/

RUN pip install \
 -r arrow/python/requirements-build.txt \
 -r arrow/python/requirements-test.txt
ENV ARROW_PYTHON_VENV /arrow-dev
RUN python3 -m venv ${ARROW_PYTHON_VENV} && \
 . ${ARROW_PYTHON_VENV}/bin/activate && \
 pip install -U pip setuptools wheel && \
 pip install \
 -r arrow/python/requirements-build.txt \
 -r arrow/python/requirements-test.txt

ARG numba
COPY ci/scripts/install_numba.sh /arrow/ci/scripts/
Expand Down

4 changes: 4 additions & 0 deletions
ci/scripts/go_cgo_python_test.sh

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -21,6 +21,10 @@ set -ex

source_dir=${1}/go

if [ -n "${ARROW_PYTHON_VENV:-}" ]; then
 . "${ARROW_PYTHON_VENV}/bin/activate"
fi

export GOFLAGS="${GOFLAGS} -gcflags=all=-d=checkptr"

pushd ${source_dir}/arrow/cdata/test
Expand Down

4 changes: 4 additions & 0 deletions
ci/scripts/install_numba.sh

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -26,6 +26,10 @@ fi

numba=$1

if [ -n "${ARROW_PYTHON_VENV:-}" ]; then
 . "${ARROW_PYTHON_VENV}/bin/activate"
fi

if [ "${numba}" = "master" ]; then
 pip install https://github.com/numba/numba/archive/main.tar.gz#egg=numba
elif [ "${numba}" = "latest" ]; then
Expand Down

4 changes: 4 additions & 0 deletions
ci/scripts/python_build.sh

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -31,6 +31,10 @@ if [ -x "$(command -v git)" ]; then
 git config --global --add safe.directory ${arrow_dir}
fi

if [ -n "${ARROW_PYTHON_VENV:-}" ]; then
 . "${ARROW_PYTHON_VENV}/bin/activate"
fi

case "$(uname)" in
 Linux)
 n_jobs=$(nproc)
Expand Down

4 changes: 4 additions & 0 deletions
ci/scripts/python_test.sh

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -22,6 +22,10 @@ set -ex
arrow_dir=${1}
test_dir=${1}/python/build/dist

if [ -n "${ARROW_PYTHON_VENV:-}" ]; then
 . "${ARROW_PYTHON_VENV}/bin/activate"
fi

export ARROW_SOURCE_DIR=${arrow_dir}
export ARROW_TEST_DATA=${arrow_dir}/testing/data
export PARQUET_TEST_DATA=${arrow_dir}/cpp/submodules/parquet-testing/data
Expand Down

2 changes: 0 additions & 2 deletions
dev/tasks/docker-tests/github.linux.yml

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -27,9 +27,7 @@ jobs:
{{ macros.github_set_env(env) }}
 steps:
 {{ macros.github_checkout_arrow(fetch_depth=fetch_depth|default(1))|indent }}
 {% if image in ["ubuntu-r-only-r"] %}
 {{ macros.github_free_space()|indent }}
 {% endif %}
 {{ macros.github_install_archery()|indent }}

 - name: Execute Docker Build
Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Strip utm_, fbclid, gclid, etc. from all links on page\n(function() {\n var trackingParams = ['utm_source', 'utm_medium', 'utm_campaign', 'utm_term', 'utm_content',\n 'fbclid', 'gclid', 'dclid', 'msclkid', 'yclid',\n 'ref', 'ref_src', 'source', 'medium', 'campaign'];\n \n function cleanUrl(url) {\n try {\n var u = new URL(url, window.location.origin);\n var changed = false;\n trackingParams.forEach(function(p) {\n if (u.searchParams.has(p)) {\n u.searchParams.delete(p);\n changed = true;\n }\n });\n return changed ? u.toString() : url;\n } catch (e) {\n return url;\n }\n }\n \n function cleanLinks() {\n document.querySelectorAll('a[href]').forEach(function(a) {\n var clean = cleanUrl(a.href);\n if (clean !== a.href) a.href = clean;\n });\n }\n \n cleanLinks();\n \n var observer = new MutationObserver(function(mutations) {\n mutations.forEach(function(m) {\n m.addedNodes.forEach(function(node) {\n if (node.nodeType === 1) {\n if (node.tagName === 'A') cleanLinks();\n node.querySelectorAll('a[href]').forEach(function(a) {\n var clean = cleanUrl(a.href);\n if (clean !== a.href) a.href = clean;\n });\n }\n });\n });\n });\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Remove Tracking Parameters from Links");
}
} catch(__e) { console.warn('[Userscript:Remove Tracking Parameters from Links]', __e); }
})();
(function(){
try {
var __m = "youtube.com";
var __re = new RegExp('^' + "youtube\\.com" + '

Skip to content
GH-40454: [CI][Debian] Update Debian to 12 from 11#40455

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

raulcd
merged 4 commits into

from

Mar 13, 2024
 Merged


GH-40454: [CI][Debian] Update Debian to 12 from 11
#40455
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


4 commits


Select commit
Hold shift + click to select a range
281c437

GH-40454: [CI][Debian] Update Debian to 12 from 11


kou Mar 11, 2024
3a5b568

Update pip


kou Mar 11, 2024
429fcda

Free space


kou Mar 12, 2024
8039954

Add missing $


kou Mar 12, 2024

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


2 changes: 1 addition & 1 deletion
.env

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -48,7 +48,7 @@ ULIMIT_CORE=-1
# Default versions for platforms
ALMALINUX=8
ALPINE_LINUX=3.16
DEBIAN=11
DEBIAN=12
FEDORA=39
UBUNTU=20.04

Expand Down

8 changes: 4 additions & 4 deletions
.github/workflows/go.yml

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -90,7 +90,7 @@ jobs:
 echo "JSON" >> "$GITHUB_OUTPUT"

 docker:
 name: ${{ matrix.arch-label }} Debian 11 Go ${{ matrix.go }}
 name: ${{ matrix.arch-label }} Debian 12 Go ${{ matrix.go }}
 needs: docker-targets
 runs-on: ${{ matrix.runs-on }}
 if: ${{ !contains(github.event.pull_request.title, 'WIP') }}
Expand DownExpand Up@@ -154,7 +154,7 @@ jobs:
 CONBENCH_EMAIL: ${{ secrets.CONBENCH_EMAIL }}
 CONBENCH_PASSWORD: ${{ secrets.CONBENCH_PASS }}
 CONBENCH_REF: ${{ github.ref_name }}
 CONBENCH_MACHINE_INFO_NAME: ${{ matrix.arch }}-debian-11
 CONBENCH_MACHINE_INFO_NAME: ${{ matrix.arch }}-debian-12
 run: |
 python3 -m pip install benchadapt@git+https://github.com/conbench/conbench.git@main#subdirectory=benchadapt/python
 python3 ci/scripts/go_bench_adapt.py
Expand All@@ -181,7 +181,7 @@ jobs:
 GOARCH=386 go build ./...

 docker_cgo:
 name: AMD64 Debian 11 Go ${{ matrix.go }} - CGO
 name: AMD64 Debian 12 Go ${{ matrix.go }} - CGO
 runs-on: ubuntu-latest
 if: ${{ !contains(github.event.pull_request.title, 'WIP') }}
 timeout-minutes: 20
Expand DownExpand Up@@ -222,7 +222,7 @@ jobs:


 docker_cgo_python:
 name: AMD64 Debian 11 Go ${{ matrix.go }} - CGO Python
 name: AMD64 Debian 12 Go ${{ matrix.go }} - CGO Python
 runs-on: ubuntu-latest
 if: ${{ !contains(github.event.pull_request.title, 'WIP') }}
 timeout-minutes: 15
Expand Down

2 changes: 1 addition & 1 deletion
.github/workflows/js.yml

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -41,7 +41,7 @@ permissions:
jobs:

 docker:
 name: AMD64 Debian 11 NodeJS 18
 name: AMD64 Debian 12 NodeJS 18
 runs-on: ubuntu-latest
 if: ${{ !contains(github.event.pull_request.title, 'WIP') }}
 timeout-minutes: 60
Expand Down

64 changes: 39 additions & 25 deletions
ci/docker/debian-11-cpp.dockerfile → ci/docker/debian-12-cpp.dockerfile

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -16,46 +16,42 @@
# under the License.

ARG arch=amd64
FROM ${arch}/debian:11
FROM ${arch}/debian:12
ARG arch

ENV DEBIAN_FRONTEND noninteractive

ARG llvm
# We can't use LLVM 14 or later from apt.llvm.org on i386 because LLVM
# 14 or later dropped support for i386.
RUN apt-get update -y -q && \
 apt-get install -y -q --no-install-recommends \
 dpkg-dev && \
 latest_available_llvm_i386=13 && \
 if [ $(dpkg-architecture -qDEB_HOST_ARCH) = "i386" -a \
 "${llvm}" -gt "${latest_available_llvm_i386}" ]; then \
 available_llvm="${latest_available_llvm_i386}"; \
 else \
 available_llvm="${llvm}"; \
 fi && \
 apt-get update -y -q && \
 apt-get install -y -q --no-install-recommends \
 apt-transport-https \
 ca-certificates \
 gnupg \
 lsb-release \
 wget && \
 wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - && \
 echo "deb https://apt.llvm.org/bullseye/ llvm-toolchain-bullseye-${available_llvm} main" > \
 /etc/apt/sources.list.d/llvm.list && \
 if [ ${llvm} -ge 17 ]; then \
 wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | \
 gpg \
 --import - \
 --keyring /usr/share/keyrings/llvm-snapshot.gpg \
 --no-default-keyring && \
 echo "deb[keyring=/usr/share/keyrings/llvm-snapshot.gpg] https://apt.llvm.org/$(lsb_release --codename --short)/ llvm-toolchain-$(lsb_release --codename --short)-${available_llvm} main" > \
 /etc/apt/sources.list.d/llvm.list; \
 fi && \
 apt-get update -y -q && \
 apt-get install -y -q --no-install-recommends \
 autoconf \
 ccache \
 clang-${available_llvm} \
 clang-${llvm} \
 cmake \
 curl \
 g++ \
 gcc \
 gdb \
 git \
 libbenchmark-dev \
 libboost-all-dev \
 libboost-filesystem-dev \
 libboost-system-dev \
 libbrotli-dev \
 libbz2-dev \
 libc-ares-dev \
Expand All@@ -64,22 +60,35 @@ RUN apt-get update -y -q && \
 libgmock-dev \
 libgoogle-glog-dev \
 libgrpc++-dev \
 libidn2-dev \
 libkrb5-dev \
 libldap-dev \
 liblz4-dev \
 libnghttp2-dev \
 libprotobuf-dev \
 libprotoc-dev \
 libpsl-dev \
 libre2-dev \
 librtmp-dev \
 libsnappy-dev \
 libsqlite3-dev \
 libssh-dev \
 libssh2-1-dev \
 libssl-dev \
 libthrift-dev \
 libutf8proc-dev \
 libxml2-dev \
 libzstd-dev \
 llvm-${available_llvm}-dev \
 llvm-${llvm}-dev \
 make \
 ninja-build \
 nlohmann-json3-dev \
 npm \
 pkg-config \
 protobuf-compiler-grpc \
 python3-dev \
 python3-pip \
 python3-venv \
 rapidjson-dev \
 rsync \
 tzdata \
Expand All@@ -93,18 +102,24 @@ RUN /arrow/ci/scripts/install_minio.sh latest /usr/local
COPY ci/scripts/install_gcs_testbench.sh /arrow/ci/scripts/
RUN /arrow/ci/scripts/install_gcs_testbench.sh default

COPY ci/scripts/install_azurite.sh /arrow/ci/scripts/
RUN /arrow/ci/scripts/install_azurite.sh

COPY ci/scripts/install_sccache.sh /arrow/ci/scripts/
RUN /arrow/ci/scripts/install_sccache.sh unknown-linux-musl /usr/local/bin

ENV absl_SOURCE=BUNDLED \
ARROW_ACERO=ON \
ENV ARROW_ACERO=ON \
ARROW_AZURE=ON \
 ARROW_BUILD_TESTS=ON \
 ARROW_DATASET=ON \
 ARROW_DEPENDENCY_SOURCE=SYSTEM \
 ARROW_DATASET=ON \
 ARROW_FLIGHT=ON \
 ARROW_FLIGHT_SQL=ON \
 ARROW_GANDIVA=ON \
 ARROW_GCS=ON \
 ARROW_HOME=/usr/local \
 ARROW_NO_DEPRECATED_API=ON \
 ARROW_ORC=ON \
 ARROW_PARQUET=ON \
 ARROW_S3=ON \
Expand All@@ -113,15 +128,14 @@ ENV absl_SOURCE=BUNDLED \
 ARROW_WITH_BROTLI=ON \
 ARROW_WITH_BZ2=ON \
 ARROW_WITH_LZ4=ON \
 ARROW_WITH_OPENTELEMETRY=OFF \
 ARROW_WITH_OPENTELEMETRY=ON \
 ARROW_WITH_SNAPPY=ON \
 ARROW_WITH_ZLIB=ON \
 ARROW_WITH_ZSTD=ON \
 AWSSDK_SOURCE=BUNDLED \
 CC=gcc \
 CXX=g++ \
 Azure_SOURCE=BUNDLED \
 google_cloud_cpp_storage_SOURCE=BUNDLED \
 GTest_SOURCE=BUNDLED \
 ORC_SOURCE=BUNDLED \
 PATH=/usr/lib/ccache/:$PATH \
 PYTHON=python3 \
 xsimd_SOURCE=BUNDLED

14 changes: 6 additions & 8 deletions
ci/docker/debian-11-go-cgo-python.dockerfile → ci/docker/debian-12-go-cgo-python.dockerfile

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -24,13 +24,11 @@ ENV DEBIAN_FRONTEND noninteractive
RUN apt-get update -y -q && \
 apt-get install -y -q --no-install-recommends \
 python3 \
 python3-pip && \
 python3-pip \
 python3-venv && \
 apt-get clean

RUN ln -s /usr/bin/python3 /usr/local/bin/python && \
 ln -s /usr/bin/pip3 /usr/local/bin/pip

# Need a newer pip than Debian's to install manylinux201x wheels
RUN pip install -U pip

RUN pip install pyarrow cffi --only-binary pyarrow
ENV ARROW_PYTHON_VENV /arrow-dev
RUN python3 -m venv ${ARROW_PYTHON_VENV} && \
 . ${ARROW_PYTHON_VENV}/bin/activate && \
 pip install pyarrow cffi --only-binary pyarrow

2 changes: 1 addition & 1 deletion
ci/docker/debian-11-go.dockerfile → ci/docker/debian-12-go.dockerfile

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -18,7 +18,7 @@
ARG arch=amd64
ARG go=1.19
ARG staticcheck=v0.4.5
FROM ${arch}/golang:${go}-bullseye
FROM ${arch}/golang:${go}-bookworm

# FROM collects all the args, get back the staticcheck version arg
ARG staticcheck
Expand Down

2 changes: 1 addition & 1 deletion
ci/docker/debian-11-js.dockerfile → ci/docker/debian-12-js.dockerfile

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -16,7 +16,7 @@
# under the License.

ARG arch=amd64
ARG node=16
ARG node=18
FROM ${arch}/node:${node}

ENV NODE_NO_WARNINGS=1
Expand Down

20 changes: 2 additions & 18 deletions
ci/docker/linux-apt-c-glib.dockerfile

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -25,6 +25,7 @@ RUN apt-get update -y -q && \
 libglib2.0-doc \
 lsb-release \
 luarocks \
 meson \
 ninja-build \
 pkg-config \
 python3 \
Expand All@@ -36,24 +37,7 @@ RUN apt-get update -y -q && \

RUN luarocks install lgi

# pip on Ubuntu 20.04 may be buggy:
#
# Collecting meson
# Downloading meson-0.53.2.tar.gz (1.6 MB)
# Installing build dependencies: started
# Installing build dependencies: finished with status 'done'
# Getting requirements to build wheel: started
# Getting requirements to build wheel: finished with status 'error'
# ERROR: Command errored out with exit status 1:
# command: /usr/bin/python3 /usr/share/python-wheels/pep517-0.7.0-py2.py3-none-any.whl/pep517/_in_process.py get_requires_for_build_wheel /tmp/tmpsk4jveay
# cwd: /tmp/pip-install-jn79a_kh/meson
# Complete output (1 lines):
# /usr/bin/python3: can't find '__main__' module in '/usr/share/python-wheels/pep517-0.7.0-py2.py3-none-any.whl/pep517/_in_process.py'
# ----------------------------------------
# ERROR: Command errored out with exit status 1: /usr/bin/python3 /usr/share/python-wheels/pep517-0.7.0-py2.py3-none-any.whl/pep517/_in_process.py get_requires_for_build_wheel /tmp/tmpsk4jveay Check the logs for full command output.
RUN (python3 -m pip install meson || \
 python3 -m pip install --no-use-pep517 meson) && \
 gem install --no-document bundler
RUN gem install --no-document bundler

COPY c_glib/Gemfile /arrow/c_glib/
RUN bundle install --gemfile /arrow/c_glib/Gemfile
Expand Down

23 changes: 7 additions & 16 deletions
ci/docker/linux-apt-python-3.dockerfile

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -18,26 +18,17 @@
ARG base
FROM ${base}

RUN apt-get update -y -q && \
 apt-get install -y -q \
 python3 \
 python3-pip \
 python3-dev && \
 apt-get clean && \
 rm -rf /var/lib/apt/lists/*

RUN ln -s /usr/bin/python3 /usr/local/bin/python && \
 ln -s /usr/bin/pip3 /usr/local/bin/pip

RUN pip install -U pip setuptools wheel

COPY python/requirements-build.txt \
 python/requirements-test.txt \
 /arrow/python/

RUN pip install \
 -r arrow/python/requirements-build.txt \
 -r arrow/python/requirements-test.txt
ENV ARROW_PYTHON_VENV /arrow-dev
RUN python3 -m venv ${ARROW_PYTHON_VENV} && \
 . ${ARROW_PYTHON_VENV}/bin/activate && \
 pip install -U pip setuptools wheel && \
 pip install \
 -r arrow/python/requirements-build.txt \
 -r arrow/python/requirements-test.txt

ARG numba
COPY ci/scripts/install_numba.sh /arrow/ci/scripts/
Expand Down

4 changes: 4 additions & 0 deletions
ci/scripts/go_cgo_python_test.sh

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -21,6 +21,10 @@ set -ex

source_dir=${1}/go

if [ -n "${ARROW_PYTHON_VENV:-}" ]; then
 . "${ARROW_PYTHON_VENV}/bin/activate"
fi

export GOFLAGS="${GOFLAGS} -gcflags=all=-d=checkptr"

pushd ${source_dir}/arrow/cdata/test
Expand Down

4 changes: 4 additions & 0 deletions
ci/scripts/install_numba.sh

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -26,6 +26,10 @@ fi

numba=$1

if [ -n "${ARROW_PYTHON_VENV:-}" ]; then
 . "${ARROW_PYTHON_VENV}/bin/activate"
fi

if [ "${numba}" = "master" ]; then
 pip install https://github.com/numba/numba/archive/main.tar.gz#egg=numba
elif [ "${numba}" = "latest" ]; then
Expand Down

4 changes: 4 additions & 0 deletions
ci/scripts/python_build.sh

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -31,6 +31,10 @@ if [ -x "$(command -v git)" ]; then
 git config --global --add safe.directory ${arrow_dir}
fi

if [ -n "${ARROW_PYTHON_VENV:-}" ]; then
 . "${ARROW_PYTHON_VENV}/bin/activate"
fi

case "$(uname)" in
 Linux)
 n_jobs=$(nproc)
Expand Down

4 changes: 4 additions & 0 deletions
ci/scripts/python_test.sh

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -22,6 +22,10 @@ set -ex
arrow_dir=${1}
test_dir=${1}/python/build/dist

if [ -n "${ARROW_PYTHON_VENV:-}" ]; then
 . "${ARROW_PYTHON_VENV}/bin/activate"
fi

export ARROW_SOURCE_DIR=${arrow_dir}
export ARROW_TEST_DATA=${arrow_dir}/testing/data
export PARQUET_TEST_DATA=${arrow_dir}/cpp/submodules/parquet-testing/data
Expand Down

2 changes: 0 additions & 2 deletions
dev/tasks/docker-tests/github.linux.yml

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -27,9 +27,7 @@ jobs:
{{ macros.github_set_env(env) }}
 steps:
 {{ macros.github_checkout_arrow(fetch_depth=fetch_depth|default(1))|indent }}
 {% if image in ["ubuntu-r-only-r"] %}
 {{ macros.github_free_space()|indent }}
 {% endif %}
 {{ macros.github_install_archery()|indent }}

 - name: Execute Docker Build
Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Auto-enable theater mode on YouTube\n(function() {\n function tryTheater() {\n var btn = document.querySelector('button[aria-label=\"Theater mode\"], ytd-player #player button[title=\"Theater mode\"]');\n if (btn && !btn.classList.contains('activated')) {\n btn.click();\n }\n }\n \n // Try immediately\n tryTheater();\n \n // Try after navigation (SPA)\n var lastUrl = location.href;\n setInterval(function() {\n if (location.href !== lastUrl) {\n lastUrl = location.href;\n setTimeout(tryTheater, 500);\n }\n }, 1000);\n \n // Also try on player load\n var observer = new MutationObserver(tryTheater);\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "YouTube Theater Mode Default");
}
} catch(__e) { console.warn('[Userscript:YouTube Theater Mode Default]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
GH-40454: [CI][Debian] Update Debian to 12 from 11#40455

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

raulcd
merged 4 commits into

from

Mar 13, 2024
 Merged


GH-40454: [CI][Debian] Update Debian to 12 from 11
#40455
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


4 commits


Select commit
Hold shift + click to select a range
281c437

GH-40454: [CI][Debian] Update Debian to 12 from 11


kou Mar 11, 2024
3a5b568

Update pip


kou Mar 11, 2024
429fcda

Free space


kou Mar 12, 2024
8039954

Add missing $


kou Mar 12, 2024

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


2 changes: 1 addition & 1 deletion
.env

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -48,7 +48,7 @@ ULIMIT_CORE=-1
# Default versions for platforms
ALMALINUX=8
ALPINE_LINUX=3.16
DEBIAN=11
DEBIAN=12
FEDORA=39
UBUNTU=20.04

Expand Down

8 changes: 4 additions & 4 deletions
.github/workflows/go.yml

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -90,7 +90,7 @@ jobs:
 echo "JSON" >> "$GITHUB_OUTPUT"

 docker:
 name: ${{ matrix.arch-label }} Debian 11 Go ${{ matrix.go }}
 name: ${{ matrix.arch-label }} Debian 12 Go ${{ matrix.go }}
 needs: docker-targets
 runs-on: ${{ matrix.runs-on }}
 if: ${{ !contains(github.event.pull_request.title, 'WIP') }}
Expand DownExpand Up@@ -154,7 +154,7 @@ jobs:
 CONBENCH_EMAIL: ${{ secrets.CONBENCH_EMAIL }}
 CONBENCH_PASSWORD: ${{ secrets.CONBENCH_PASS }}
 CONBENCH_REF: ${{ github.ref_name }}
 CONBENCH_MACHINE_INFO_NAME: ${{ matrix.arch }}-debian-11
 CONBENCH_MACHINE_INFO_NAME: ${{ matrix.arch }}-debian-12
 run: |
 python3 -m pip install benchadapt@git+https://github.com/conbench/conbench.git@main#subdirectory=benchadapt/python
 python3 ci/scripts/go_bench_adapt.py
Expand All@@ -181,7 +181,7 @@ jobs:
 GOARCH=386 go build ./...

 docker_cgo:
 name: AMD64 Debian 11 Go ${{ matrix.go }} - CGO
 name: AMD64 Debian 12 Go ${{ matrix.go }} - CGO
 runs-on: ubuntu-latest
 if: ${{ !contains(github.event.pull_request.title, 'WIP') }}
 timeout-minutes: 20
Expand DownExpand Up@@ -222,7 +222,7 @@ jobs:


 docker_cgo_python:
 name: AMD64 Debian 11 Go ${{ matrix.go }} - CGO Python
 name: AMD64 Debian 12 Go ${{ matrix.go }} - CGO Python
 runs-on: ubuntu-latest
 if: ${{ !contains(github.event.pull_request.title, 'WIP') }}
 timeout-minutes: 15
Expand Down

2 changes: 1 addition & 1 deletion
.github/workflows/js.yml

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -41,7 +41,7 @@ permissions:
jobs:

 docker:
 name: AMD64 Debian 11 NodeJS 18
 name: AMD64 Debian 12 NodeJS 18
 runs-on: ubuntu-latest
 if: ${{ !contains(github.event.pull_request.title, 'WIP') }}
 timeout-minutes: 60
Expand Down

64 changes: 39 additions & 25 deletions
ci/docker/debian-11-cpp.dockerfile → ci/docker/debian-12-cpp.dockerfile

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -16,46 +16,42 @@
# under the License.

ARG arch=amd64
FROM ${arch}/debian:11
FROM ${arch}/debian:12
ARG arch

ENV DEBIAN_FRONTEND noninteractive

ARG llvm
# We can't use LLVM 14 or later from apt.llvm.org on i386 because LLVM
# 14 or later dropped support for i386.
RUN apt-get update -y -q && \
 apt-get install -y -q --no-install-recommends \
 dpkg-dev && \
 latest_available_llvm_i386=13 && \
 if [ $(dpkg-architecture -qDEB_HOST_ARCH) = "i386" -a \
 "${llvm}" -gt "${latest_available_llvm_i386}" ]; then \
 available_llvm="${latest_available_llvm_i386}"; \
 else \
 available_llvm="${llvm}"; \
 fi && \
 apt-get update -y -q && \
 apt-get install -y -q --no-install-recommends \
 apt-transport-https \
 ca-certificates \
 gnupg \
 lsb-release \
 wget && \
 wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - && \
 echo "deb https://apt.llvm.org/bullseye/ llvm-toolchain-bullseye-${available_llvm} main" > \
 /etc/apt/sources.list.d/llvm.list && \
 if [ ${llvm} -ge 17 ]; then \
 wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | \
 gpg \
 --import - \
 --keyring /usr/share/keyrings/llvm-snapshot.gpg \
 --no-default-keyring && \
 echo "deb[keyring=/usr/share/keyrings/llvm-snapshot.gpg] https://apt.llvm.org/$(lsb_release --codename --short)/ llvm-toolchain-$(lsb_release --codename --short)-${available_llvm} main" > \
 /etc/apt/sources.list.d/llvm.list; \
 fi && \
 apt-get update -y -q && \
 apt-get install -y -q --no-install-recommends \
 autoconf \
 ccache \
 clang-${available_llvm} \
 clang-${llvm} \
 cmake \
 curl \
 g++ \
 gcc \
 gdb \
 git \
 libbenchmark-dev \
 libboost-all-dev \
 libboost-filesystem-dev \
 libboost-system-dev \
 libbrotli-dev \
 libbz2-dev \
 libc-ares-dev \
Expand All@@ -64,22 +60,35 @@ RUN apt-get update -y -q && \
 libgmock-dev \
 libgoogle-glog-dev \
 libgrpc++-dev \
 libidn2-dev \
 libkrb5-dev \
 libldap-dev \
 liblz4-dev \
 libnghttp2-dev \
 libprotobuf-dev \
 libprotoc-dev \
 libpsl-dev \
 libre2-dev \
 librtmp-dev \
 libsnappy-dev \
 libsqlite3-dev \
 libssh-dev \
 libssh2-1-dev \
 libssl-dev \
 libthrift-dev \
 libutf8proc-dev \
 libxml2-dev \
 libzstd-dev \
 llvm-${available_llvm}-dev \
 llvm-${llvm}-dev \
 make \
 ninja-build \
 nlohmann-json3-dev \
 npm \
 pkg-config \
 protobuf-compiler-grpc \
 python3-dev \
 python3-pip \
 python3-venv \
 rapidjson-dev \
 rsync \
 tzdata \
Expand All@@ -93,18 +102,24 @@ RUN /arrow/ci/scripts/install_minio.sh latest /usr/local
COPY ci/scripts/install_gcs_testbench.sh /arrow/ci/scripts/
RUN /arrow/ci/scripts/install_gcs_testbench.sh default

COPY ci/scripts/install_azurite.sh /arrow/ci/scripts/
RUN /arrow/ci/scripts/install_azurite.sh

COPY ci/scripts/install_sccache.sh /arrow/ci/scripts/
RUN /arrow/ci/scripts/install_sccache.sh unknown-linux-musl /usr/local/bin

ENV absl_SOURCE=BUNDLED \
ARROW_ACERO=ON \
ENV ARROW_ACERO=ON \
ARROW_AZURE=ON \
 ARROW_BUILD_TESTS=ON \
 ARROW_DATASET=ON \
 ARROW_DEPENDENCY_SOURCE=SYSTEM \
 ARROW_DATASET=ON \
 ARROW_FLIGHT=ON \
 ARROW_FLIGHT_SQL=ON \
 ARROW_GANDIVA=ON \
 ARROW_GCS=ON \
 ARROW_HOME=/usr/local \
 ARROW_NO_DEPRECATED_API=ON \
 ARROW_ORC=ON \
 ARROW_PARQUET=ON \
 ARROW_S3=ON \
Expand All@@ -113,15 +128,14 @@ ENV absl_SOURCE=BUNDLED \
 ARROW_WITH_BROTLI=ON \
 ARROW_WITH_BZ2=ON \
 ARROW_WITH_LZ4=ON \
 ARROW_WITH_OPENTELEMETRY=OFF \
 ARROW_WITH_OPENTELEMETRY=ON \
 ARROW_WITH_SNAPPY=ON \
 ARROW_WITH_ZLIB=ON \
 ARROW_WITH_ZSTD=ON \
 AWSSDK_SOURCE=BUNDLED \
 CC=gcc \
 CXX=g++ \
 Azure_SOURCE=BUNDLED \
 google_cloud_cpp_storage_SOURCE=BUNDLED \
 GTest_SOURCE=BUNDLED \
 ORC_SOURCE=BUNDLED \
 PATH=/usr/lib/ccache/:$PATH \
 PYTHON=python3 \
 xsimd_SOURCE=BUNDLED

14 changes: 6 additions & 8 deletions
ci/docker/debian-11-go-cgo-python.dockerfile → ci/docker/debian-12-go-cgo-python.dockerfile

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -24,13 +24,11 @@ ENV DEBIAN_FRONTEND noninteractive
RUN apt-get update -y -q && \
 apt-get install -y -q --no-install-recommends \
 python3 \
 python3-pip && \
 python3-pip \
 python3-venv && \
 apt-get clean

RUN ln -s /usr/bin/python3 /usr/local/bin/python && \
 ln -s /usr/bin/pip3 /usr/local/bin/pip

# Need a newer pip than Debian's to install manylinux201x wheels
RUN pip install -U pip

RUN pip install pyarrow cffi --only-binary pyarrow
ENV ARROW_PYTHON_VENV /arrow-dev
RUN python3 -m venv ${ARROW_PYTHON_VENV} && \
 . ${ARROW_PYTHON_VENV}/bin/activate && \
 pip install pyarrow cffi --only-binary pyarrow

2 changes: 1 addition & 1 deletion
ci/docker/debian-11-go.dockerfile → ci/docker/debian-12-go.dockerfile

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -18,7 +18,7 @@
ARG arch=amd64
ARG go=1.19
ARG staticcheck=v0.4.5
FROM ${arch}/golang:${go}-bullseye
FROM ${arch}/golang:${go}-bookworm

# FROM collects all the args, get back the staticcheck version arg
ARG staticcheck
Expand Down

2 changes: 1 addition & 1 deletion
ci/docker/debian-11-js.dockerfile → ci/docker/debian-12-js.dockerfile

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -16,7 +16,7 @@
# under the License.

ARG arch=amd64
ARG node=16
ARG node=18
FROM ${arch}/node:${node}

ENV NODE_NO_WARNINGS=1
Expand Down

20 changes: 2 additions & 18 deletions
ci/docker/linux-apt-c-glib.dockerfile

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -25,6 +25,7 @@ RUN apt-get update -y -q && \
 libglib2.0-doc \
 lsb-release \
 luarocks \
 meson \
 ninja-build \
 pkg-config \
 python3 \
Expand All@@ -36,24 +37,7 @@ RUN apt-get update -y -q && \

RUN luarocks install lgi

# pip on Ubuntu 20.04 may be buggy:
#
# Collecting meson
# Downloading meson-0.53.2.tar.gz (1.6 MB)
# Installing build dependencies: started
# Installing build dependencies: finished with status 'done'
# Getting requirements to build wheel: started
# Getting requirements to build wheel: finished with status 'error'
# ERROR: Command errored out with exit status 1:
# command: /usr/bin/python3 /usr/share/python-wheels/pep517-0.7.0-py2.py3-none-any.whl/pep517/_in_process.py get_requires_for_build_wheel /tmp/tmpsk4jveay
# cwd: /tmp/pip-install-jn79a_kh/meson
# Complete output (1 lines):
# /usr/bin/python3: can't find '__main__' module in '/usr/share/python-wheels/pep517-0.7.0-py2.py3-none-any.whl/pep517/_in_process.py'
# ----------------------------------------
# ERROR: Command errored out with exit status 1: /usr/bin/python3 /usr/share/python-wheels/pep517-0.7.0-py2.py3-none-any.whl/pep517/_in_process.py get_requires_for_build_wheel /tmp/tmpsk4jveay Check the logs for full command output.
RUN (python3 -m pip install meson || \
 python3 -m pip install --no-use-pep517 meson) && \
 gem install --no-document bundler
RUN gem install --no-document bundler

COPY c_glib/Gemfile /arrow/c_glib/
RUN bundle install --gemfile /arrow/c_glib/Gemfile
Expand Down

23 changes: 7 additions & 16 deletions
ci/docker/linux-apt-python-3.dockerfile

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -18,26 +18,17 @@
ARG base
FROM ${base}

RUN apt-get update -y -q && \
 apt-get install -y -q \
 python3 \
 python3-pip \
 python3-dev && \
 apt-get clean && \
 rm -rf /var/lib/apt/lists/*

RUN ln -s /usr/bin/python3 /usr/local/bin/python && \
 ln -s /usr/bin/pip3 /usr/local/bin/pip

RUN pip install -U pip setuptools wheel

COPY python/requirements-build.txt \
 python/requirements-test.txt \
 /arrow/python/

RUN pip install \
 -r arrow/python/requirements-build.txt \
 -r arrow/python/requirements-test.txt
ENV ARROW_PYTHON_VENV /arrow-dev
RUN python3 -m venv ${ARROW_PYTHON_VENV} && \
 . ${ARROW_PYTHON_VENV}/bin/activate && \
 pip install -U pip setuptools wheel && \
 pip install \
 -r arrow/python/requirements-build.txt \
 -r arrow/python/requirements-test.txt

ARG numba
COPY ci/scripts/install_numba.sh /arrow/ci/scripts/
Expand Down

4 changes: 4 additions & 0 deletions
ci/scripts/go_cgo_python_test.sh

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -21,6 +21,10 @@ set -ex

source_dir=${1}/go

if [ -n "${ARROW_PYTHON_VENV:-}" ]; then
 . "${ARROW_PYTHON_VENV}/bin/activate"
fi

export GOFLAGS="${GOFLAGS} -gcflags=all=-d=checkptr"

pushd ${source_dir}/arrow/cdata/test
Expand Down

4 changes: 4 additions & 0 deletions
ci/scripts/install_numba.sh

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -26,6 +26,10 @@ fi

numba=$1

if [ -n "${ARROW_PYTHON_VENV:-}" ]; then
 . "${ARROW_PYTHON_VENV}/bin/activate"
fi

if [ "${numba}" = "master" ]; then
 pip install https://github.com/numba/numba/archive/main.tar.gz#egg=numba
elif [ "${numba}" = "latest" ]; then
Expand Down

4 changes: 4 additions & 0 deletions
ci/scripts/python_build.sh

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -31,6 +31,10 @@ if [ -x "$(command -v git)" ]; then
 git config --global --add safe.directory ${arrow_dir}
fi

if [ -n "${ARROW_PYTHON_VENV:-}" ]; then
 . "${ARROW_PYTHON_VENV}/bin/activate"
fi

case "$(uname)" in
 Linux)
 n_jobs=$(nproc)
Expand Down

4 changes: 4 additions & 0 deletions
ci/scripts/python_test.sh

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -22,6 +22,10 @@ set -ex
arrow_dir=${1}
test_dir=${1}/python/build/dist

if [ -n "${ARROW_PYTHON_VENV:-}" ]; then
 . "${ARROW_PYTHON_VENV}/bin/activate"
fi

export ARROW_SOURCE_DIR=${arrow_dir}
export ARROW_TEST_DATA=${arrow_dir}/testing/data
export PARQUET_TEST_DATA=${arrow_dir}/cpp/submodules/parquet-testing/data
Expand Down

2 changes: 0 additions & 2 deletions
dev/tasks/docker-tests/github.linux.yml

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -27,9 +27,7 @@ jobs:
{{ macros.github_set_env(env) }}
 steps:
 {{ macros.github_checkout_arrow(fetch_depth=fetch_depth|default(1))|indent }}
 {% if image in ["ubuntu-r-only-r"] %}
 {{ macros.github_free_space()|indent }}
 {% endif %}
 {{ macros.github_install_archery()|indent }}

 - name: Execute Docker Build
Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Remove or un-stick sticky/fixed headers that block content\n(function() {\n function unstick() {\n document.querySelectorAll('header, nav, [role=\"banner\"], .header, .navbar, .sticky, .fixed-top, [style*=\"position: fixed\"], [style*=\"position:sticky\"]').forEach(function(el) {\n if (el.style.position === 'fixed' || el.style.position === 'sticky' || \n getComputedStyle(el).position === 'fixed' || getComputedStyle(el).position === 'sticky') {\n el.style.position = 'static';\n el.style.top = 'auto';\n el.style.zIndex = 'auto';\n }\n });\n }\n \n unstick();\n \n var observer = new MutationObserver(unstick);\n observer.observe(document.body, { childList: true, subtree: true, attributes: true, attributeFilter: ['style', 'class'] });\n})();", "Kill Sticky Headers");
}
} catch(__e) { console.warn('[Userscript:Kill Sticky Headers]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
GH-40454: [CI][Debian] Update Debian to 12 from 11#40455

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

raulcd
merged 4 commits into

from

Mar 13, 2024
 Merged


GH-40454: [CI][Debian] Update Debian to 12 from 11
#40455
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


4 commits


Select commit
Hold shift + click to select a range
281c437

GH-40454: [CI][Debian] Update Debian to 12 from 11


kou Mar 11, 2024
3a5b568

Update pip


kou Mar 11, 2024
429fcda

Free space


kou Mar 12, 2024
8039954

Add missing $


kou Mar 12, 2024

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


2 changes: 1 addition & 1 deletion
.env

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -48,7 +48,7 @@ ULIMIT_CORE=-1
# Default versions for platforms
ALMALINUX=8
ALPINE_LINUX=3.16
DEBIAN=11
DEBIAN=12
FEDORA=39
UBUNTU=20.04

Expand Down

8 changes: 4 additions & 4 deletions
.github/workflows/go.yml

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -90,7 +90,7 @@ jobs:
 echo "JSON" >> "$GITHUB_OUTPUT"

 docker:
 name: ${{ matrix.arch-label }} Debian 11 Go ${{ matrix.go }}
 name: ${{ matrix.arch-label }} Debian 12 Go ${{ matrix.go }}
 needs: docker-targets
 runs-on: ${{ matrix.runs-on }}
 if: ${{ !contains(github.event.pull_request.title, 'WIP') }}
Expand DownExpand Up@@ -154,7 +154,7 @@ jobs:
 CONBENCH_EMAIL: ${{ secrets.CONBENCH_EMAIL }}
 CONBENCH_PASSWORD: ${{ secrets.CONBENCH_PASS }}
 CONBENCH_REF: ${{ github.ref_name }}
 CONBENCH_MACHINE_INFO_NAME: ${{ matrix.arch }}-debian-11
 CONBENCH_MACHINE_INFO_NAME: ${{ matrix.arch }}-debian-12
 run: |
 python3 -m pip install benchadapt@git+https://github.com/conbench/conbench.git@main#subdirectory=benchadapt/python
 python3 ci/scripts/go_bench_adapt.py
Expand All@@ -181,7 +181,7 @@ jobs:
 GOARCH=386 go build ./...

 docker_cgo:
 name: AMD64 Debian 11 Go ${{ matrix.go }} - CGO
 name: AMD64 Debian 12 Go ${{ matrix.go }} - CGO
 runs-on: ubuntu-latest
 if: ${{ !contains(github.event.pull_request.title, 'WIP') }}
 timeout-minutes: 20
Expand DownExpand Up@@ -222,7 +222,7 @@ jobs:


 docker_cgo_python:
 name: AMD64 Debian 11 Go ${{ matrix.go }} - CGO Python
 name: AMD64 Debian 12 Go ${{ matrix.go }} - CGO Python
 runs-on: ubuntu-latest
 if: ${{ !contains(github.event.pull_request.title, 'WIP') }}
 timeout-minutes: 15
Expand Down

2 changes: 1 addition & 1 deletion
.github/workflows/js.yml

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -41,7 +41,7 @@ permissions:
jobs:

 docker:
 name: AMD64 Debian 11 NodeJS 18
 name: AMD64 Debian 12 NodeJS 18
 runs-on: ubuntu-latest
 if: ${{ !contains(github.event.pull_request.title, 'WIP') }}
 timeout-minutes: 60
Expand Down

64 changes: 39 additions & 25 deletions
ci/docker/debian-11-cpp.dockerfile → ci/docker/debian-12-cpp.dockerfile

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -16,46 +16,42 @@
# under the License.

ARG arch=amd64
FROM ${arch}/debian:11
FROM ${arch}/debian:12
ARG arch

ENV DEBIAN_FRONTEND noninteractive

ARG llvm
# We can't use LLVM 14 or later from apt.llvm.org on i386 because LLVM
# 14 or later dropped support for i386.
RUN apt-get update -y -q && \
 apt-get install -y -q --no-install-recommends \
 dpkg-dev && \
 latest_available_llvm_i386=13 && \
 if [ $(dpkg-architecture -qDEB_HOST_ARCH) = "i386" -a \
 "${llvm}" -gt "${latest_available_llvm_i386}" ]; then \
 available_llvm="${latest_available_llvm_i386}"; \
 else \
 available_llvm="${llvm}"; \
 fi && \
 apt-get update -y -q && \
 apt-get install -y -q --no-install-recommends \
 apt-transport-https \
 ca-certificates \
 gnupg \
 lsb-release \
 wget && \
 wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - && \
 echo "deb https://apt.llvm.org/bullseye/ llvm-toolchain-bullseye-${available_llvm} main" > \
 /etc/apt/sources.list.d/llvm.list && \
 if [ ${llvm} -ge 17 ]; then \
 wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | \
 gpg \
 --import - \
 --keyring /usr/share/keyrings/llvm-snapshot.gpg \
 --no-default-keyring && \
 echo "deb[keyring=/usr/share/keyrings/llvm-snapshot.gpg] https://apt.llvm.org/$(lsb_release --codename --short)/ llvm-toolchain-$(lsb_release --codename --short)-${available_llvm} main" > \
 /etc/apt/sources.list.d/llvm.list; \
 fi && \
 apt-get update -y -q && \
 apt-get install -y -q --no-install-recommends \
 autoconf \
 ccache \
 clang-${available_llvm} \
 clang-${llvm} \
 cmake \
 curl \
 g++ \
 gcc \
 gdb \
 git \
 libbenchmark-dev \
 libboost-all-dev \
 libboost-filesystem-dev \
 libboost-system-dev \
 libbrotli-dev \
 libbz2-dev \
 libc-ares-dev \
Expand All@@ -64,22 +60,35 @@ RUN apt-get update -y -q && \
 libgmock-dev \
 libgoogle-glog-dev \
 libgrpc++-dev \
 libidn2-dev \
 libkrb5-dev \
 libldap-dev \
 liblz4-dev \
 libnghttp2-dev \
 libprotobuf-dev \
 libprotoc-dev \
 libpsl-dev \
 libre2-dev \
 librtmp-dev \
 libsnappy-dev \
 libsqlite3-dev \
 libssh-dev \
 libssh2-1-dev \
 libssl-dev \
 libthrift-dev \
 libutf8proc-dev \
 libxml2-dev \
 libzstd-dev \
 llvm-${available_llvm}-dev \
 llvm-${llvm}-dev \
 make \
 ninja-build \
 nlohmann-json3-dev \
 npm \
 pkg-config \
 protobuf-compiler-grpc \
 python3-dev \
 python3-pip \
 python3-venv \
 rapidjson-dev \
 rsync \
 tzdata \
Expand All@@ -93,18 +102,24 @@ RUN /arrow/ci/scripts/install_minio.sh latest /usr/local
COPY ci/scripts/install_gcs_testbench.sh /arrow/ci/scripts/
RUN /arrow/ci/scripts/install_gcs_testbench.sh default

COPY ci/scripts/install_azurite.sh /arrow/ci/scripts/
RUN /arrow/ci/scripts/install_azurite.sh

COPY ci/scripts/install_sccache.sh /arrow/ci/scripts/
RUN /arrow/ci/scripts/install_sccache.sh unknown-linux-musl /usr/local/bin

ENV absl_SOURCE=BUNDLED \
ARROW_ACERO=ON \
ENV ARROW_ACERO=ON \
ARROW_AZURE=ON \
 ARROW_BUILD_TESTS=ON \
 ARROW_DATASET=ON \
 ARROW_DEPENDENCY_SOURCE=SYSTEM \
 ARROW_DATASET=ON \
 ARROW_FLIGHT=ON \
 ARROW_FLIGHT_SQL=ON \
 ARROW_GANDIVA=ON \
 ARROW_GCS=ON \
 ARROW_HOME=/usr/local \
 ARROW_NO_DEPRECATED_API=ON \
 ARROW_ORC=ON \
 ARROW_PARQUET=ON \
 ARROW_S3=ON \
Expand All@@ -113,15 +128,14 @@ ENV absl_SOURCE=BUNDLED \
 ARROW_WITH_BROTLI=ON \
 ARROW_WITH_BZ2=ON \
 ARROW_WITH_LZ4=ON \
 ARROW_WITH_OPENTELEMETRY=OFF \
 ARROW_WITH_OPENTELEMETRY=ON \
 ARROW_WITH_SNAPPY=ON \
 ARROW_WITH_ZLIB=ON \
 ARROW_WITH_ZSTD=ON \
 AWSSDK_SOURCE=BUNDLED \
 CC=gcc \
 CXX=g++ \
 Azure_SOURCE=BUNDLED \
 google_cloud_cpp_storage_SOURCE=BUNDLED \
 GTest_SOURCE=BUNDLED \
 ORC_SOURCE=BUNDLED \
 PATH=/usr/lib/ccache/:$PATH \
 PYTHON=python3 \
 xsimd_SOURCE=BUNDLED

14 changes: 6 additions & 8 deletions
ci/docker/debian-11-go-cgo-python.dockerfile → ci/docker/debian-12-go-cgo-python.dockerfile

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -24,13 +24,11 @@ ENV DEBIAN_FRONTEND noninteractive
RUN apt-get update -y -q && \
 apt-get install -y -q --no-install-recommends \
 python3 \
 python3-pip && \
 python3-pip \
 python3-venv && \
 apt-get clean

RUN ln -s /usr/bin/python3 /usr/local/bin/python && \
 ln -s /usr/bin/pip3 /usr/local/bin/pip

# Need a newer pip than Debian's to install manylinux201x wheels
RUN pip install -U pip

RUN pip install pyarrow cffi --only-binary pyarrow
ENV ARROW_PYTHON_VENV /arrow-dev
RUN python3 -m venv ${ARROW_PYTHON_VENV} && \
 . ${ARROW_PYTHON_VENV}/bin/activate && \
 pip install pyarrow cffi --only-binary pyarrow

2 changes: 1 addition & 1 deletion
ci/docker/debian-11-go.dockerfile → ci/docker/debian-12-go.dockerfile

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -18,7 +18,7 @@
ARG arch=amd64
ARG go=1.19
ARG staticcheck=v0.4.5
FROM ${arch}/golang:${go}-bullseye
FROM ${arch}/golang:${go}-bookworm

# FROM collects all the args, get back the staticcheck version arg
ARG staticcheck
Expand Down

2 changes: 1 addition & 1 deletion
ci/docker/debian-11-js.dockerfile → ci/docker/debian-12-js.dockerfile

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -16,7 +16,7 @@
# under the License.

ARG arch=amd64
ARG node=16
ARG node=18
FROM ${arch}/node:${node}

ENV NODE_NO_WARNINGS=1
Expand Down

20 changes: 2 additions & 18 deletions
ci/docker/linux-apt-c-glib.dockerfile

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -25,6 +25,7 @@ RUN apt-get update -y -q && \
 libglib2.0-doc \
 lsb-release \
 luarocks \
 meson \
 ninja-build \
 pkg-config \
 python3 \
Expand All@@ -36,24 +37,7 @@ RUN apt-get update -y -q && \

RUN luarocks install lgi

# pip on Ubuntu 20.04 may be buggy:
#
# Collecting meson
# Downloading meson-0.53.2.tar.gz (1.6 MB)
# Installing build dependencies: started
# Installing build dependencies: finished with status 'done'
# Getting requirements to build wheel: started
# Getting requirements to build wheel: finished with status 'error'
# ERROR: Command errored out with exit status 1:
# command: /usr/bin/python3 /usr/share/python-wheels/pep517-0.7.0-py2.py3-none-any.whl/pep517/_in_process.py get_requires_for_build_wheel /tmp/tmpsk4jveay
# cwd: /tmp/pip-install-jn79a_kh/meson
# Complete output (1 lines):
# /usr/bin/python3: can't find '__main__' module in '/usr/share/python-wheels/pep517-0.7.0-py2.py3-none-any.whl/pep517/_in_process.py'
# ----------------------------------------
# ERROR: Command errored out with exit status 1: /usr/bin/python3 /usr/share/python-wheels/pep517-0.7.0-py2.py3-none-any.whl/pep517/_in_process.py get_requires_for_build_wheel /tmp/tmpsk4jveay Check the logs for full command output.
RUN (python3 -m pip install meson || \
 python3 -m pip install --no-use-pep517 meson) && \
 gem install --no-document bundler
RUN gem install --no-document bundler

COPY c_glib/Gemfile /arrow/c_glib/
RUN bundle install --gemfile /arrow/c_glib/Gemfile
Expand Down

23 changes: 7 additions & 16 deletions
ci/docker/linux-apt-python-3.dockerfile

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -18,26 +18,17 @@
ARG base
FROM ${base}

RUN apt-get update -y -q && \
 apt-get install -y -q \
 python3 \
 python3-pip \
 python3-dev && \
 apt-get clean && \
 rm -rf /var/lib/apt/lists/*

RUN ln -s /usr/bin/python3 /usr/local/bin/python && \
 ln -s /usr/bin/pip3 /usr/local/bin/pip

RUN pip install -U pip setuptools wheel

COPY python/requirements-build.txt \
 python/requirements-test.txt \
 /arrow/python/

RUN pip install \
 -r arrow/python/requirements-build.txt \
 -r arrow/python/requirements-test.txt
ENV ARROW_PYTHON_VENV /arrow-dev
RUN python3 -m venv ${ARROW_PYTHON_VENV} && \
 . ${ARROW_PYTHON_VENV}/bin/activate && \
 pip install -U pip setuptools wheel && \
 pip install \
 -r arrow/python/requirements-build.txt \
 -r arrow/python/requirements-test.txt

ARG numba
COPY ci/scripts/install_numba.sh /arrow/ci/scripts/
Expand Down

4 changes: 4 additions & 0 deletions
ci/scripts/go_cgo_python_test.sh

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -21,6 +21,10 @@ set -ex

source_dir=${1}/go

if [ -n "${ARROW_PYTHON_VENV:-}" ]; then
 . "${ARROW_PYTHON_VENV}/bin/activate"
fi

export GOFLAGS="${GOFLAGS} -gcflags=all=-d=checkptr"

pushd ${source_dir}/arrow/cdata/test
Expand Down

4 changes: 4 additions & 0 deletions
ci/scripts/install_numba.sh

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -26,6 +26,10 @@ fi

numba=$1

if [ -n "${ARROW_PYTHON_VENV:-}" ]; then
 . "${ARROW_PYTHON_VENV}/bin/activate"
fi

if [ "${numba}" = "master" ]; then
 pip install https://github.com/numba/numba/archive/main.tar.gz#egg=numba
elif [ "${numba}" = "latest" ]; then
Expand Down

4 changes: 4 additions & 0 deletions
ci/scripts/python_build.sh

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -31,6 +31,10 @@ if [ -x "$(command -v git)" ]; then
 git config --global --add safe.directory ${arrow_dir}
fi

if [ -n "${ARROW_PYTHON_VENV:-}" ]; then
 . "${ARROW_PYTHON_VENV}/bin/activate"
fi

case "$(uname)" in
 Linux)
 n_jobs=$(nproc)
Expand Down

4 changes: 4 additions & 0 deletions
ci/scripts/python_test.sh

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -22,6 +22,10 @@ set -ex
arrow_dir=${1}
test_dir=${1}/python/build/dist

if [ -n "${ARROW_PYTHON_VENV:-}" ]; then
 . "${ARROW_PYTHON_VENV}/bin/activate"
fi

export ARROW_SOURCE_DIR=${arrow_dir}
export ARROW_TEST_DATA=${arrow_dir}/testing/data
export PARQUET_TEST_DATA=${arrow_dir}/cpp/submodules/parquet-testing/data
Expand Down

2 changes: 0 additions & 2 deletions
dev/tasks/docker-tests/github.linux.yml

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -27,9 +27,7 @@ jobs:
{{ macros.github_set_env(env) }}
 steps:
 {{ macros.github_checkout_arrow(fetch_depth=fetch_depth|default(1))|indent }}
 {% if image in ["ubuntu-r-only-r"] %}
 {{ macros.github_free_space()|indent }}
 {% endif %}
 {{ macros.github_install_archery()|indent }}

 - name: Execute Docker Build
Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Universal Dark Mode - works on any site\n(function() {\n var enabled = true;\n \n function applyDarkMode() {\n if (!enabled) return;\n \n // Create style element if it doesn't exist\n var style = document.getElementById('universal-dark-mode-style');\n if (!style) {\n style = document.createElement('style');\n style.id = 'universal-dark-mode-style';\n document.head.appendChild(style);\n }\n \n // Dark mode CSS - inverts colors but preserves images/video\n style.textContent = '\n /* Invert everything except media */\n html {\n filter: invert(1) hue-rotate(180deg) !important;\n background: #1a1a2e !important;\n }\n \n /* Restore images, videos, iframes, canvas */\n img, video, iframe, canvas, svg, picture, [style*=\"background-image\"] {\n filter: invert(1) hue-rotate(180deg) !important;\n }\n \n /* Preserve specific elements that should not be inverted */\n .no-dark-mode, .no-dark-mode *,\n [data-theme=\"light\"], [data-theme=\"light\"],\n .ace_editor, .ace_editor *,\n .CodeMirror, .CodeMirror *,\n .monaco-editor, .monaco-editor *,\n .markdown-body pre, .markdown-body pre *,\n .highlight, .highlight *,\n pre code, pre code * {\n filter: none !important;\n }\n \n /* Fix common UI elements */\n .modal, .popup, .dropdown-menu, .tooltip, .popover {\n filter: invert(1) hue-rotate(180deg) !important;\n background: #2d2d44 !important;\n border-color: #444 !important;\n }\n \n /* Scrollbars */\n ::-webkit-scrollbar { background: #1a1a2e !important; }\n ::-webkit-scrollbar-thumb { background: #444 !important; }\n ::-webkit-scrollbar-thumb:hover { background: #555 !important; }\n \n /* Selection */\n ::selection { background: #4ecdc4 !important; color: #1a1a2e !important; }\n ::-moz-selection { background: #4ecdc4 !important; color: #1a1a2e !important; }\n ';\n }\n \n function removeDarkMode() {\n var style = document.getElementById('universal-dark-mode-style');\n if (style) style.remove();\n }\n \n // Toggle with Alt+Shift+D\n document.addEventListener('keydown', function(e) {\n if (e.altKey && e.shiftKey && e.key === 'D') {\n e.preventDefault();\n enabled = !enabled;\n if (enabled) {\n applyDarkMode();\n console.log('[Universal Dark Mode] Enabled');\n } else {\n removeDarkMode();\n console.log('[Universal Dark Mode] Disabled');\n }\n }\n });\n \n // Apply on load\n applyDarkMode();\n \n // Re-apply on dynamic content\n var observer = new MutationObserver(function(mutations) {\n if (enabled && !document.getElementById('universal-dark-mode-style')) {\n applyDarkMode();\n }\n });\n observer.observe(document.head, { childList: true });\n \n console.log('[Universal Dark Mode] Loaded - Press Alt+Shift+D to toggle');\n})();", "Universal Dark Mode");
}
} catch(__e) { console.warn('[Userscript:Universal Dark Mode]', __e); }
})();
})();

Skip to content
GH-40454: [CI][Debian] Update Debian to 12 from 11#40455

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

raulcd
merged 4 commits into

from

Mar 13, 2024
 Merged


GH-40454: [CI][Debian] Update Debian to 12 from 11
#40455
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


4 commits


Select commit
Hold shift + click to select a range
281c437

GH-40454: [CI][Debian] Update Debian to 12 from 11


kou Mar 11, 2024
3a5b568

Update pip


kou Mar 11, 2024
429fcda

Free space


kou Mar 12, 2024
8039954

Add missing $


kou Mar 12, 2024

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


2 changes: 1 addition & 1 deletion
.env

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -48,7 +48,7 @@ ULIMIT_CORE=-1
# Default versions for platforms
ALMALINUX=8
ALPINE_LINUX=3.16
DEBIAN=11
DEBIAN=12
FEDORA=39
UBUNTU=20.04

Expand Down

8 changes: 4 additions & 4 deletions
.github/workflows/go.yml

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -90,7 +90,7 @@ jobs:
 echo "JSON" >> "$GITHUB_OUTPUT"

 docker:
 name: ${{ matrix.arch-label }} Debian 11 Go ${{ matrix.go }}
 name: ${{ matrix.arch-label }} Debian 12 Go ${{ matrix.go }}
 needs: docker-targets
 runs-on: ${{ matrix.runs-on }}
 if: ${{ !contains(github.event.pull_request.title, 'WIP') }}
Expand DownExpand Up@@ -154,7 +154,7 @@ jobs:
 CONBENCH_EMAIL: ${{ secrets.CONBENCH_EMAIL }}
 CONBENCH_PASSWORD: ${{ secrets.CONBENCH_PASS }}
 CONBENCH_REF: ${{ github.ref_name }}
 CONBENCH_MACHINE_INFO_NAME: ${{ matrix.arch }}-debian-11
 CONBENCH_MACHINE_INFO_NAME: ${{ matrix.arch }}-debian-12
 run: |
 python3 -m pip install benchadapt@git+https://github.com/conbench/conbench.git@main#subdirectory=benchadapt/python
 python3 ci/scripts/go_bench_adapt.py
Expand All@@ -181,7 +181,7 @@ jobs:
 GOARCH=386 go build ./...

 docker_cgo:
 name: AMD64 Debian 11 Go ${{ matrix.go }} - CGO
 name: AMD64 Debian 12 Go ${{ matrix.go }} - CGO
 runs-on: ubuntu-latest
 if: ${{ !contains(github.event.pull_request.title, 'WIP') }}
 timeout-minutes: 20
Expand DownExpand Up@@ -222,7 +222,7 @@ jobs:


 docker_cgo_python:
 name: AMD64 Debian 11 Go ${{ matrix.go }} - CGO Python
 name: AMD64 Debian 12 Go ${{ matrix.go }} - CGO Python
 runs-on: ubuntu-latest
 if: ${{ !contains(github.event.pull_request.title, 'WIP') }}
 timeout-minutes: 15
Expand Down

2 changes: 1 addition & 1 deletion
.github/workflows/js.yml

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -41,7 +41,7 @@ permissions:
jobs:

 docker:
 name: AMD64 Debian 11 NodeJS 18
 name: AMD64 Debian 12 NodeJS 18
 runs-on: ubuntu-latest
 if: ${{ !contains(github.event.pull_request.title, 'WIP') }}
 timeout-minutes: 60
Expand Down

64 changes: 39 additions & 25 deletions
ci/docker/debian-11-cpp.dockerfile → ci/docker/debian-12-cpp.dockerfile

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -16,46 +16,42 @@
# under the License.

ARG arch=amd64
FROM ${arch}/debian:11
FROM ${arch}/debian:12
ARG arch

ENV DEBIAN_FRONTEND noninteractive

ARG llvm
# We can't use LLVM 14 or later from apt.llvm.org on i386 because LLVM
# 14 or later dropped support for i386.
RUN apt-get update -y -q && \
 apt-get install -y -q --no-install-recommends \
 dpkg-dev && \
 latest_available_llvm_i386=13 && \
 if [ $(dpkg-architecture -qDEB_HOST_ARCH) = "i386" -a \
 "${llvm}" -gt "${latest_available_llvm_i386}" ]; then \
 available_llvm="${latest_available_llvm_i386}"; \
 else \
 available_llvm="${llvm}"; \
 fi && \
 apt-get update -y -q && \
 apt-get install -y -q --no-install-recommends \
 apt-transport-https \
 ca-certificates \
 gnupg \
 lsb-release \
 wget && \
 wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - && \
 echo "deb https://apt.llvm.org/bullseye/ llvm-toolchain-bullseye-${available_llvm} main" > \
 /etc/apt/sources.list.d/llvm.list && \
 if [ ${llvm} -ge 17 ]; then \
 wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | \
 gpg \
 --import - \
 --keyring /usr/share/keyrings/llvm-snapshot.gpg \
 --no-default-keyring && \
 echo "deb[keyring=/usr/share/keyrings/llvm-snapshot.gpg] https://apt.llvm.org/$(lsb_release --codename --short)/ llvm-toolchain-$(lsb_release --codename --short)-${available_llvm} main" > \
 /etc/apt/sources.list.d/llvm.list; \
 fi && \
 apt-get update -y -q && \
 apt-get install -y -q --no-install-recommends \
 autoconf \
 ccache \
 clang-${available_llvm} \
 clang-${llvm} \
 cmake \
 curl \
 g++ \
 gcc \
 gdb \
 git \
 libbenchmark-dev \
 libboost-all-dev \
 libboost-filesystem-dev \
 libboost-system-dev \
 libbrotli-dev \
 libbz2-dev \
 libc-ares-dev \
Expand All@@ -64,22 +60,35 @@ RUN apt-get update -y -q && \
 libgmock-dev \
 libgoogle-glog-dev \
 libgrpc++-dev \
 libidn2-dev \
 libkrb5-dev \
 libldap-dev \
 liblz4-dev \
 libnghttp2-dev \
 libprotobuf-dev \
 libprotoc-dev \
 libpsl-dev \
 libre2-dev \
 librtmp-dev \
 libsnappy-dev \
 libsqlite3-dev \
 libssh-dev \
 libssh2-1-dev \
 libssl-dev \
 libthrift-dev \
 libutf8proc-dev \
 libxml2-dev \
 libzstd-dev \
 llvm-${available_llvm}-dev \
 llvm-${llvm}-dev \
 make \
 ninja-build \
 nlohmann-json3-dev \
 npm \
 pkg-config \
 protobuf-compiler-grpc \
 python3-dev \
 python3-pip \
 python3-venv \
 rapidjson-dev \
 rsync \
 tzdata \
Expand All@@ -93,18 +102,24 @@ RUN /arrow/ci/scripts/install_minio.sh latest /usr/local
COPY ci/scripts/install_gcs_testbench.sh /arrow/ci/scripts/
RUN /arrow/ci/scripts/install_gcs_testbench.sh default

COPY ci/scripts/install_azurite.sh /arrow/ci/scripts/
RUN /arrow/ci/scripts/install_azurite.sh

COPY ci/scripts/install_sccache.sh /arrow/ci/scripts/
RUN /arrow/ci/scripts/install_sccache.sh unknown-linux-musl /usr/local/bin

ENV absl_SOURCE=BUNDLED \
ARROW_ACERO=ON \
ENV ARROW_ACERO=ON \
ARROW_AZURE=ON \
 ARROW_BUILD_TESTS=ON \
 ARROW_DATASET=ON \
 ARROW_DEPENDENCY_SOURCE=SYSTEM \
 ARROW_DATASET=ON \
 ARROW_FLIGHT=ON \
 ARROW_FLIGHT_SQL=ON \
 ARROW_GANDIVA=ON \
 ARROW_GCS=ON \
 ARROW_HOME=/usr/local \
 ARROW_NO_DEPRECATED_API=ON \
 ARROW_ORC=ON \
 ARROW_PARQUET=ON \
 ARROW_S3=ON \
Expand All@@ -113,15 +128,14 @@ ENV absl_SOURCE=BUNDLED \
 ARROW_WITH_BROTLI=ON \
 ARROW_WITH_BZ2=ON \
 ARROW_WITH_LZ4=ON \
 ARROW_WITH_OPENTELEMETRY=OFF \
 ARROW_WITH_OPENTELEMETRY=ON \
 ARROW_WITH_SNAPPY=ON \
 ARROW_WITH_ZLIB=ON \
 ARROW_WITH_ZSTD=ON \
 AWSSDK_SOURCE=BUNDLED \
 CC=gcc \
 CXX=g++ \
 Azure_SOURCE=BUNDLED \
 google_cloud_cpp_storage_SOURCE=BUNDLED \
 GTest_SOURCE=BUNDLED \
 ORC_SOURCE=BUNDLED \
 PATH=/usr/lib/ccache/:$PATH \
 PYTHON=python3 \
 xsimd_SOURCE=BUNDLED

14 changes: 6 additions & 8 deletions
ci/docker/debian-11-go-cgo-python.dockerfile → ci/docker/debian-12-go-cgo-python.dockerfile

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -24,13 +24,11 @@ ENV DEBIAN_FRONTEND noninteractive
RUN apt-get update -y -q && \
 apt-get install -y -q --no-install-recommends \
 python3 \
 python3-pip && \
 python3-pip \
 python3-venv && \
 apt-get clean

RUN ln -s /usr/bin/python3 /usr/local/bin/python && \
 ln -s /usr/bin/pip3 /usr/local/bin/pip

# Need a newer pip than Debian's to install manylinux201x wheels
RUN pip install -U pip

RUN pip install pyarrow cffi --only-binary pyarrow
ENV ARROW_PYTHON_VENV /arrow-dev
RUN python3 -m venv ${ARROW_PYTHON_VENV} && \
 . ${ARROW_PYTHON_VENV}/bin/activate && \
 pip install pyarrow cffi --only-binary pyarrow

2 changes: 1 addition & 1 deletion
ci/docker/debian-11-go.dockerfile → ci/docker/debian-12-go.dockerfile

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -18,7 +18,7 @@
ARG arch=amd64
ARG go=1.19
ARG staticcheck=v0.4.5
FROM ${arch}/golang:${go}-bullseye
FROM ${arch}/golang:${go}-bookworm

# FROM collects all the args, get back the staticcheck version arg
ARG staticcheck
Expand Down

2 changes: 1 addition & 1 deletion
ci/docker/debian-11-js.dockerfile → ci/docker/debian-12-js.dockerfile

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -16,7 +16,7 @@
# under the License.

ARG arch=amd64
ARG node=16
ARG node=18
FROM ${arch}/node:${node}

ENV NODE_NO_WARNINGS=1
Expand Down

20 changes: 2 additions & 18 deletions
ci/docker/linux-apt-c-glib.dockerfile

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -25,6 +25,7 @@ RUN apt-get update -y -q && \
 libglib2.0-doc \
 lsb-release \
 luarocks \
 meson \
 ninja-build \
 pkg-config \
 python3 \
Expand All@@ -36,24 +37,7 @@ RUN apt-get update -y -q && \

RUN luarocks install lgi

# pip on Ubuntu 20.04 may be buggy:
#
# Collecting meson
# Downloading meson-0.53.2.tar.gz (1.6 MB)
# Installing build dependencies: started
# Installing build dependencies: finished with status 'done'
# Getting requirements to build wheel: started
# Getting requirements to build wheel: finished with status 'error'
# ERROR: Command errored out with exit status 1:
# command: /usr/bin/python3 /usr/share/python-wheels/pep517-0.7.0-py2.py3-none-any.whl/pep517/_in_process.py get_requires_for_build_wheel /tmp/tmpsk4jveay
# cwd: /tmp/pip-install-jn79a_kh/meson
# Complete output (1 lines):
# /usr/bin/python3: can't find '__main__' module in '/usr/share/python-wheels/pep517-0.7.0-py2.py3-none-any.whl/pep517/_in_process.py'
# ----------------------------------------
# ERROR: Command errored out with exit status 1: /usr/bin/python3 /usr/share/python-wheels/pep517-0.7.0-py2.py3-none-any.whl/pep517/_in_process.py get_requires_for_build_wheel /tmp/tmpsk4jveay Check the logs for full command output.
RUN (python3 -m pip install meson || \
 python3 -m pip install --no-use-pep517 meson) && \
 gem install --no-document bundler
RUN gem install --no-document bundler

COPY c_glib/Gemfile /arrow/c_glib/
RUN bundle install --gemfile /arrow/c_glib/Gemfile
Expand Down

23 changes: 7 additions & 16 deletions
ci/docker/linux-apt-python-3.dockerfile

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -18,26 +18,17 @@
ARG base
FROM ${base}

RUN apt-get update -y -q && \
 apt-get install -y -q \
 python3 \
 python3-pip \
 python3-dev && \
 apt-get clean && \
 rm -rf /var/lib/apt/lists/*

RUN ln -s /usr/bin/python3 /usr/local/bin/python && \
 ln -s /usr/bin/pip3 /usr/local/bin/pip

RUN pip install -U pip setuptools wheel

COPY python/requirements-build.txt \
 python/requirements-test.txt \
 /arrow/python/

RUN pip install \
 -r arrow/python/requirements-build.txt \
 -r arrow/python/requirements-test.txt
ENV ARROW_PYTHON_VENV /arrow-dev
RUN python3 -m venv ${ARROW_PYTHON_VENV} && \
 . ${ARROW_PYTHON_VENV}/bin/activate && \
 pip install -U pip setuptools wheel && \
 pip install \
 -r arrow/python/requirements-build.txt \
 -r arrow/python/requirements-test.txt

ARG numba
COPY ci/scripts/install_numba.sh /arrow/ci/scripts/
Expand Down

4 changes: 4 additions & 0 deletions
ci/scripts/go_cgo_python_test.sh

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -21,6 +21,10 @@ set -ex

source_dir=${1}/go

if [ -n "${ARROW_PYTHON_VENV:-}" ]; then
 . "${ARROW_PYTHON_VENV}/bin/activate"
fi

export GOFLAGS="${GOFLAGS} -gcflags=all=-d=checkptr"

pushd ${source_dir}/arrow/cdata/test
Expand Down

4 changes: 4 additions & 0 deletions
ci/scripts/install_numba.sh

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -26,6 +26,10 @@ fi

numba=$1

if [ -n "${ARROW_PYTHON_VENV:-}" ]; then
 . "${ARROW_PYTHON_VENV}/bin/activate"
fi

if [ "${numba}" = "master" ]; then
 pip install https://github.com/numba/numba/archive/main.tar.gz#egg=numba
elif [ "${numba}" = "latest" ]; then
Expand Down

4 changes: 4 additions & 0 deletions
ci/scripts/python_build.sh

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -31,6 +31,10 @@ if [ -x "$(command -v git)" ]; then
 git config --global --add safe.directory ${arrow_dir}
fi

if [ -n "${ARROW_PYTHON_VENV:-}" ]; then
 . "${ARROW_PYTHON_VENV}/bin/activate"
fi

case "$(uname)" in
 Linux)
 n_jobs=$(nproc)
Expand Down

4 changes: 4 additions & 0 deletions
ci/scripts/python_test.sh

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -22,6 +22,10 @@ set -ex
arrow_dir=${1}
test_dir=${1}/python/build/dist

if [ -n "${ARROW_PYTHON_VENV:-}" ]; then
 . "${ARROW_PYTHON_VENV}/bin/activate"
fi

export ARROW_SOURCE_DIR=${arrow_dir}
export ARROW_TEST_DATA=${arrow_dir}/testing/data
export PARQUET_TEST_DATA=${arrow_dir}/cpp/submodules/parquet-testing/data
Expand Down

2 changes: 0 additions & 2 deletions
dev/tasks/docker-tests/github.linux.yml

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -27,9 +27,7 @@ jobs:
{{ macros.github_set_env(env) }}
 steps:
 {{ macros.github_checkout_arrow(fetch_depth=fetch_depth|default(1))|indent }}
 {% if image in ["ubuntu-r-only-r"] %}
 {{ macros.github_free_space()|indent }}
 {% endif %}
 {{ macros.github_install_archery()|indent }}

 - name: Execute Docker Build
Expand Down
Loading