diff --git a/packaging/src/docker/Dockerfile b/packaging/src/docker/Dockerfile index b400533506e3..ba77deefcf1b 100644 --- a/packaging/src/docker/Dockerfile +++ b/packaging/src/docker/Dockerfile @@ -67,7 +67,8 @@ RUN tar -xzv \ tar -xzv \ --exclude="apache-tez-$TEZ_VERSION-bin/share" \ -f /opt/apache-tez-$TEZ_VERSION-bin.tar.gz \ - -C /opt + -C /opt; \ + mkdir -p /opt/tez-snapshot; FROM eclipse-temurin:21-jdk-ubi9-minimal AS run @@ -75,10 +76,13 @@ ARG UID=1000 ARG HADOOP_VERSION ARG HIVE_VERSION ARG TEZ_VERSION +ARG TEZ_SNAPSHOT_VERSION= +ARG TEZ_SNAPSHOT_REPO_URL=https://repository.apache.org/content/repositories/snapshots + # Install dependencies RUN set -ex; \ microdnf update -y; \ - microdnf -y install procps gettext; \ + microdnf -y install procps gettext wget xmlstarlet; \ microdnf clean all; \ useradd --no-create-home -s /sbin/nologin -c "" --uid $UID hive @@ -93,6 +97,38 @@ ENV PATH=$HIVE_HOME/bin:$HADOOP_HOME/bin:$PATH COPY --from=env --chown=hive /opt/hadoop-$HADOOP_VERSION $HADOOP_HOME COPY --from=env --chown=hive /opt/apache-hive-$HIVE_VERSION-bin $HIVE_HOME COPY --from=env --chown=hive /opt/apache-tez-$TEZ_VERSION-bin $TEZ_HOME +COPY --from=env --chown=hive /opt/tez-snapshot /opt/tez-snapshot + +# When TEZ_SNAPSHOT_VERSION is set, fetch Tez snapshot jars from the Maven snapshot repository +# and place them under /opt/tez-snapshot. At runtime, entrypoint.sh symlinks these into +# $HIVE_HOME/lib with a "0-" prefix so they sort first in bin/hive's classpath glob, ensuring +# snapshot classes take precedence over the Tez release jars bundled with Hive. +# Maven snapshot repositories use timestamped filenames (e.g. tez-api-1.0.0-20250101.jar), +# so we fetch maven-metadata.xml first to resolve the exact filename before downloading the jar. +RUN set -eux; \ + mkdir -p /opt/tez-snapshot-download; \ + if [[ -n "${TEZ_SNAPSHOT_VERSION}" ]]; then \ + base_url="${TEZ_SNAPSHOT_REPO_URL}/org/apache/tez"; \ + for artifact in tez-common tez-api tez-dag tez-mapreduce tez-runtime-internals tez-runtime-library; do \ + version_url="${base_url}/${artifact}/${TEZ_SNAPSHOT_VERSION}"; \ + metadata_url="${version_url}/maven-metadata.xml"; \ + metadata_file="/opt/tez-snapshot-download/${artifact}-maven-metadata.xml"; \ + echo "metadata_url=${metadata_url}"; \ + wget -q "${metadata_url}" -O "${metadata_file}"; \ + snapshot_value="$(xmlstarlet sel -t -v "string(/metadata/versioning/snapshotVersions/snapshotVersion[extension='jar' and not(classifier)]/value)" "${metadata_file}")"; \ + test -n "${snapshot_value}"; \ + jar_file="${artifact}-${snapshot_value}.jar"; \ + jar_url="${version_url}/${jar_file}"; \ + echo "jar_url=${jar_url}"; \ + wget -q "${jar_url}" -O "/opt/tez-snapshot/${jar_file}"; \ + done; \ + echo "Downloaded Tez snapshot jars under /opt/tez-snapshot:"; \ + ls -1 /opt/tez-snapshot/*.jar; \ + else \ + echo "TEZ_SNAPSHOT_VERSION not set. Skipping Tez snapshot download."; \ + fi; \ + rm -rf /opt/tez-snapshot-download + COPY --chown=hive entrypoint.sh / COPY --chown=hive conf $HIVE_HOME/conf @@ -103,7 +139,8 @@ RUN chmod +x /entrypoint.sh && \ mkdir -p $HIVE_HOME/scratch && \ chown hive $HIVE_HOME/scratch && \ mkdir -p /home/hive/.beeline && \ - chown hive /home/hive/.beeline + chown hive /home/hive/.beeline && \ + chown -R hive /opt/tez-snapshot USER hive WORKDIR $HIVE_HOME diff --git a/packaging/src/docker/build.sh b/packaging/src/docker/build.sh index cd6747da53ec..356e1bcd6c4a 100755 --- a/packaging/src/docker/build.sh +++ b/packaging/src/docker/build.sh @@ -20,13 +20,15 @@ set -eux HIVE_VERSION= HADOOP_VERSION= TEZ_VERSION= +TEZ_SNAPSHOT_VERSION= usage() { cat <&2 -Usage: $0 [-h] [-hadoop ] -tez [-hive ] [-repo ] +Usage: $0 [-h] [-hadoop ] -tez [-tez-snapshot []] [-hive ] [-repo ] Build the Hive Docker image (reused for LLAP too) -help Display help -hadoop Build image with the specified Hadoop version (default: from Maven pom) -tez Required. Tez release tarball version (apache-tez-\$TEZ_VERSION-bin.tar.gz from archive) +-tez-snapshot Optional. When a snapshot version is given, fetch Tez Maven snapshot jars into the image. With no version, snapshot prefetch is skipped. -hive Build image with the specified Hive version -repo Docker repository EOF @@ -48,6 +50,13 @@ while [ $# -gt 0 ]; do TEZ_VERSION=$1 shift ;; + -tez-snapshot) + shift + if [ $# -gt 0 ] && [[ "$1" != -* ]]; then + TEZ_SNAPSHOT_VERSION=$1 + shift + fi + ;; -hive) shift HIVE_VERSION=$1 @@ -135,6 +144,9 @@ DOCKER_BUILD_ARGS=( --build-arg "HADOOP_VERSION=$HADOOP_VERSION" --build-arg "TEZ_VERSION=$TEZ_VERSION" ) +if [ -n "$TEZ_SNAPSHOT_VERSION" ]; then + DOCKER_BUILD_ARGS+=(--build-arg "TEZ_SNAPSHOT_VERSION=$TEZ_SNAPSHOT_VERSION") +fi docker build \ "$WORK_DIR" \ diff --git a/packaging/src/docker/entrypoint.sh b/packaging/src/docker/entrypoint.sh index 377fe594d6e2..d06cd06dabbc 100644 --- a/packaging/src/docker/entrypoint.sh +++ b/packaging/src/docker/entrypoint.sh @@ -147,8 +147,13 @@ function run_tezam { export HIVE_HOME="${HIVE_HOME:-/opt/hive}" export HADOOP_CONF_DIR="${HADOOP_CONF_DIR:-$HIVE_CONF_DIR}" export TEZ_CONF_DIR="${TEZ_CONF_DIR:-$HADOOP_CONF_DIR}" + : "${TEZ_SNAPSHOT_HOME:=/opt/tez-snapshot}" + if [[ ! -d "${TEZ_SNAPSHOT_HOME}" ]]; then + echo "Tez snapshot home not found at ${TEZ_SNAPSHOT_HOME}. Rebuild image to prefetch snapshot artifacts." + exit 1 + fi # service_plugins_descriptor.json references org.apache.hadoop.hive.llap.tezplugins.* (hive-llap-tez, etc.) - tezam_cp="${HADOOP_CONF_DIR}:${TEZ_CONF_DIR}:${TEZ_HOME}/*:${TEZ_HOME}/lib/*:${HIVE_HOME}/lib/*:${HADOOP_HOME}/share/hadoop/common/*:${HADOOP_HOME}/share/hadoop/common/lib/*:${HADOOP_HOME}/share/hadoop/yarn/*:${HADOOP_HOME}/share/hadoop/yarn/lib/*:${HADOOP_HOME}/share/hadoop/hdfs/*:${HADOOP_HOME}/share/hadoop/hdfs/lib/*:${HADOOP_HOME}/share/hadoop/mapreduce/*:${HADOOP_HOME}/share/hadoop/mapreduce/lib/*:${HADOOP_CLASSPATH:-}" + tezam_cp="${HADOOP_CONF_DIR}:${TEZ_CONF_DIR}:${TEZ_SNAPSHOT_HOME}/*:${TEZ_HOME}/*:${TEZ_HOME}/lib/*:${HIVE_HOME}/lib/*:${HADOOP_HOME}/share/hadoop/common/*:${HADOOP_HOME}/share/hadoop/common/lib/*:${HADOOP_HOME}/share/hadoop/yarn/*:${HADOOP_HOME}/share/hadoop/yarn/lib/*:${HADOOP_HOME}/share/hadoop/hdfs/*:${HADOOP_HOME}/share/hadoop/hdfs/lib/*:${HADOOP_HOME}/share/hadoop/mapreduce/*:${HADOOP_HOME}/share/hadoop/mapreduce/lib/*:${HADOOP_CLASSPATH:-}" local java_bin local tezam_java_opts @@ -178,6 +183,17 @@ if [[ "${SKIP_SCHEMA_INIT}" == "false" && ( "${SERVICE_NAME}" == "hiveserver2" | fi if [ "${SERVICE_NAME}" == "hiveserver2" ]; then + TEZ_SNAPSHOT_HOME="${TEZ_SNAPSHOT_HOME:-/opt/tez-snapshot}" + # bin/hive prepends all of $HIVE_HOME/lib/*.jar to $HADOOP_CLASSPATH, so any entry + # we put first in HADOOP_CLASSPATH ends up after all Hive lib jars. To get snapshot jars + # truly first, symlink them into $HIVE_HOME/lib/ with a "0-" prefix: the for-loop glob in + # bin/hive processes jars alphabetically, and ASCII '0' (48) sorts before 'a' (97), so these + # symlinks become the very first jars on the classpath (right after the conf dir entries). + if ls "${TEZ_SNAPSHOT_HOME}"/*.jar 1>/dev/null 2>&1; then + for snap_jar in "${TEZ_SNAPSHOT_HOME}"/*.jar; do + ln -sf "$snap_jar" "${HIVE_HOME}/lib/0-$(basename "$snap_jar")" + done + fi export HADOOP_CLASSPATH="$TEZ_HOME/*:$TEZ_HOME/lib/*:$HADOOP_CLASSPATH" exec "$HIVE_HOME/bin/hive" --skiphadoopversion --skiphbasecp --service "$SERVICE_NAME" elif [ "${SERVICE_NAME}" == "metastore" ]; then