From e3459ae8b7d403ee7219887baa3b0196d4e0f5bd Mon Sep 17 00:00:00 2001 From: Laszlo Bodor Date: Thu, 23 Apr 2026 12:01:00 +0200 Subject: [PATCH] HIVE-29419: Provide a Hive-specific docker image for Tez AM --- .../llap/tezplugins/LlapTaskCommunicator.java | 2 +- packaging/src/docker/Dockerfile | 1 - packaging/src/docker/build.sh | 23 +++-- .../src/docker/conf/hive-site.xml.template | 21 +++++ .../conf/service_plugins_descriptor.json | 22 +++++ .../src/docker/conf/tez-log4j.properties | 5 ++ .../src/docker/conf/tez-site.xml.template | 74 ++++++++++++++++ packaging/src/docker/docker-compose.yml | 37 ++++++-- packaging/src/docker/entrypoint.sh | 88 ++++++++++++++----- 9 files changed, 235 insertions(+), 38 deletions(-) create mode 100644 packaging/src/docker/conf/service_plugins_descriptor.json create mode 100644 packaging/src/docker/conf/tez-log4j.properties create mode 100644 packaging/src/docker/conf/tez-site.xml.template diff --git a/llap-tez/src/java/org/apache/hadoop/hive/llap/tezplugins/LlapTaskCommunicator.java b/llap-tez/src/java/org/apache/hadoop/hive/llap/tezplugins/LlapTaskCommunicator.java index d850caae9581..0451848f606c 100644 --- a/llap-tez/src/java/org/apache/hadoop/hive/llap/tezplugins/LlapTaskCommunicator.java +++ b/llap-tez/src/java/org/apache/hadoop/hive/llap/tezplugins/LlapTaskCommunicator.java @@ -257,7 +257,7 @@ protected void startRpcServer() { int numHandlers = HiveConf.getIntVar(conf, ConfVars.LLAP_TASK_COMMUNICATOR_LISTENER_THREAD_COUNT); String[] portRange = - conf.get(HiveConf.ConfVars.LLAP_TASK_UMBILICAL_SERVER_PORT.varname) + HiveConf.getVar(conf, HiveConf.ConfVars.LLAP_TASK_UMBILICAL_SERVER_PORT) .split("-"); boolean isHadoopSecurityAuthorizationEnabled = conf.getBoolean( CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION, false); diff --git a/packaging/src/docker/Dockerfile b/packaging/src/docker/Dockerfile index 17f7c96eade8..b400533506e3 100644 --- a/packaging/src/docker/Dockerfile +++ b/packaging/src/docker/Dockerfile @@ -75,7 +75,6 @@ ARG UID=1000 ARG HADOOP_VERSION ARG HIVE_VERSION ARG TEZ_VERSION - # Install dependencies RUN set -ex; \ microdnf update -y; \ diff --git a/packaging/src/docker/build.sh b/packaging/src/docker/build.sh index 9d8258320f5f..cd6747da53ec 100755 --- a/packaging/src/docker/build.sh +++ b/packaging/src/docker/build.sh @@ -22,11 +22,11 @@ HADOOP_VERSION= TEZ_VERSION= usage() { cat <&2 -Usage: $0 [-h] [-hadoop ] [-tez ] [-hive ] [-repo ] +Usage: $0 [-h] [-hadoop ] -tez [-hive ] [-repo ] Build the Hive Docker image (reused for LLAP too) -help Display help --hadoop Build image with the specified Hadoop version --tez Build image with the specified Tez version +-hadoop Build image with the specified Hadoop version (default: from Maven pom) +-tez Required. Tez release tarball version (apache-tez-\$TEZ_VERSION-bin.tar.gz from archive) -hive Build image with the specified Hive version -repo Docker repository EOF @@ -64,6 +64,12 @@ while [ $# -gt 0 ]; do esac done +if [ -z "${TEZ_VERSION}" ]; then + echo "Error: -tez is required." >&2 + usage + exit 1 +fi + SCRIPT_DIR=$(cd $(dirname $0); pwd) SOURCE_DIR=${SOURCE_DIR:-"$SCRIPT_DIR/../../.."} repo=${REPO:-apache} @@ -123,12 +129,17 @@ cp "$CACHE_DIR/apache-tez-$TEZ_VERSION-bin.tar.gz" "$WORK_DIR/" cp -R "$SOURCE_DIR/packaging/src/docker/conf" "$WORK_DIR/" cp -R "$SOURCE_DIR/packaging/src/docker/entrypoint.sh" "$WORK_DIR/" cp "$SOURCE_DIR/packaging/src/docker/Dockerfile" "$WORK_DIR/" + +DOCKER_BUILD_ARGS=( + --build-arg "HIVE_VERSION=$HIVE_VERSION" + --build-arg "HADOOP_VERSION=$HADOOP_VERSION" + --build-arg "TEZ_VERSION=$TEZ_VERSION" +) + docker build \ "$WORK_DIR" \ -f "$WORK_DIR/Dockerfile" \ -t "$repo/hive:$HIVE_VERSION" \ - --build-arg "HIVE_VERSION=$HIVE_VERSION" \ - --build-arg "HADOOP_VERSION=$HADOOP_VERSION" \ - --build-arg "TEZ_VERSION=$TEZ_VERSION" + "${DOCKER_BUILD_ARGS[@]}" rm -r "${WORK_DIR}" diff --git a/packaging/src/docker/conf/hive-site.xml.template b/packaging/src/docker/conf/hive-site.xml.template index c3d28b29691a..13f6b8a0c09b 100644 --- a/packaging/src/docker/conf/hive-site.xml.template +++ b/packaging/src/docker/conf/hive-site.xml.template @@ -88,4 +88,25 @@ hive.query.results.cache.directory ${HIVE_QUERY_RESULTS_CACHE_DIRECTORY} + + hive.server2.tez.initialize.default.sessions + false + + + hive.server2.tez.use.external.sessions + ${HIVE_SERVER2_TEZ_USE_EXTERNAL_SESSIONS} + + + + hive.server2.tez.external.sessions.namespace + /tez-external-sessions${TEZ_AM_REGISTRY_NAMESPACE} + + + hive.server2.tez.external.sessions.registry.class + org.apache.hadoop.hive.ql.exec.tez.ZookeeperExternalSessionsRegistryClient + diff --git a/packaging/src/docker/conf/service_plugins_descriptor.json b/packaging/src/docker/conf/service_plugins_descriptor.json new file mode 100644 index 000000000000..133f6eaa649c --- /dev/null +++ b/packaging/src/docker/conf/service_plugins_descriptor.json @@ -0,0 +1,22 @@ +{ + "taskSchedulerDescriptors": [ + { + "className": "org.apache.hadoop.hive.llap.tezplugins.LlapTaskSchedulerService", + "entityName": "LLAP" + } + ], + "containerLauncherDescriptors": [ + { + "className": "org.apache.hadoop.hive.llap.tezplugins.LlapContainerLauncher", + "entityName": "LLAP" + } + ], + "taskCommunicatorDescriptors": [ + { + "className": "org.apache.hadoop.hive.llap.tezplugins.LlapTaskCommunicator", + "entityName": "LLAP" + } + ], + "enableContainers": false, + "enableUber": false +} diff --git a/packaging/src/docker/conf/tez-log4j.properties b/packaging/src/docker/conf/tez-log4j.properties new file mode 100644 index 000000000000..6afa36a63569 --- /dev/null +++ b/packaging/src/docker/conf/tez-log4j.properties @@ -0,0 +1,5 @@ +log4j.rootLogger=INFO, console +log4j.appender.console=org.apache.log4j.ConsoleAppender +log4j.appender.console.Target=System.err +log4j.appender.console.layout=org.apache.log4j.PatternLayout +log4j.appender.console.layout.ConversionPattern=%d{ISO8601} %-5p %c{1} - %m%n diff --git a/packaging/src/docker/conf/tez-site.xml.template b/packaging/src/docker/conf/tez-site.xml.template new file mode 100644 index 000000000000..a420137a97e2 --- /dev/null +++ b/packaging/src/docker/conf/tez-site.xml.template @@ -0,0 +1,74 @@ + + + + + + tez.am.mode.session + true + + + tez.am.framework.mode + ${TEZ_FRAMEWORK_MODE} + + + tez.am.zookeeper.quorum + ${TEZ_AM_ZOOKEEPER_QUORUM} + + + tez.am.registry.namespace + ${TEZ_AM_REGISTRY_NAMESPACE} + + + tez.local.mode + false + + + tez.am.tez-ui.webservice.enable + false + + + + tez.session.am.dag.submit.timeout.secs + -1 + + + tez.ignore.lib.uris + true + + + tez.am.disable.client-version-check + true + + + + + hive.zookeeper.quorum + ${HIVE_ZOOKEEPER_QUORUM} + + + hive.llap.daemon.service.hosts + ${HIVE_LLAP_DAEMON_SERVICE_HOSTS} + + diff --git a/packaging/src/docker/docker-compose.yml b/packaging/src/docker/docker-compose.yml index 37b9d70e1381..475b9e6669e4 100644 --- a/packaging/src/docker/docker-compose.yml +++ b/packaging/src/docker/docker-compose.yml @@ -132,14 +132,35 @@ services: - zookeeper_datalog:/datalog - zookeeper_logs:/logs - #TODO Tez AM container (in the meantime, the HS2(with local Tez AM) + LLAP daemon setup is working properly) - # 1. Define and use a Tez AM image from HIVE-29419 or TEZ-4682 - # 2. Configure TezAM to use Zookeeper Llap Registry to discover the LLAP daemon - # 3. Configure HiveServer2 to use the Tez AM Zookeeper Registry to discover the Tez AM - # Prerequisites: - # - tez-api 1.0.0-SNAPSHOT jar injected into HiveSever2 until Tez 1.0.0 is released - # - make HIVE-29477 happen to let HiveServer2 use Tez external sessions - # 4. Define hadoop components here to be used by all the containers (working example can be found at TEZ-4682), currently a local volume + tezam: + profiles: + - llap + image: apache/hive:${HIVE_VERSION} + container_name: tezam + hostname: tezam + depends_on: + - zookeeper + restart: on-failure:3 + environment: + USER: hive + SERVICE_NAME: 'tezam' + + TEZ_FRAMEWORK_MODE: STANDALONE_ZOOKEEPER + TEZ_AM_ZOOKEEPER_QUORUM: zookeeper:2181 + + # LLAP daemon discovery + HIVE_ZOOKEEPER_QUORUM: zookeeper:2181 + HIVE_LLAP_DAEMON_SERVICE_HOSTS: '@llap0' + + # Directories shared between HiveServer2 and LLAP daemon + HIVE_SCRATCH_DIR: /opt/hive/scratch + HIVE_QUERY_RESULTS_CACHE_DIRECTORY: /opt/hive/scratch/_resultscache_ + + volumes: + - warehouse:/opt/hive/data/warehouse + - scratch:/opt/hive/scratch + networks: + - hive llapdaemon: profiles: diff --git a/packaging/src/docker/entrypoint.sh b/packaging/src/docker/entrypoint.sh index 5a5979b9ac9f..377fe594d6e2 100644 --- a/packaging/src/docker/entrypoint.sh +++ b/packaging/src/docker/entrypoint.sh @@ -41,8 +41,17 @@ export HIVE_WAREHOUSE_PATH="${HIVE_WAREHOUSE_PATH:-/opt/hive/data/warehouse}" export HIVE_SCRATCH_DIR="${HIVE_SCRATCH_DIR:-/opt/hive/scratch}" export HIVE_QUERY_RESULTS_CACHE_DIRECTORY="${HIVE_WAREHOUSE_PATH:-/opt/hive/scratch/_resultscache_}" +export HIVE_ZOOKEEPER_QUORUM="${HIVE_ZOOKEEPER_QUORUM:-zookeeper:2181}" +export HIVE_LLAP_DAEMON_SERVICE_HOSTS="${HIVE_LLAP_DAEMON_SERVICE_HOSTS:-@llap0}" + +export HIVE_SERVER2_TEZ_USE_EXTERNAL_SESSIONS="${HIVE_SERVER2_TEZ_USE_EXTERNAL_SESSIONS:-true}" +export TEZ_FRAMEWORK_MODE="${TEZ_FRAMEWORK_MODE:-STANDALONE_ZOOKEEPER}" +export TEZ_AM_REGISTRY_NAMESPACE="${TEZ_AM_REGISTRY_NAMESPACE:-/tez_am/server}" +export TEZ_AM_ZOOKEEPER_QUORUM="${TEZ_AM_ZOOKEEPER_QUORUM:-${HIVE_ZOOKEEPER_QUORUM}}" + envsubst < $HIVE_HOME/conf/core-site.xml.template > $HIVE_HOME/conf/core-site.xml envsubst < $HIVE_HOME/conf/hive-site.xml.template > $HIVE_HOME/conf/hive-site.xml +envsubst < $HIVE_HOME/conf/tez-site.xml.template > $HIVE_HOME/conf/tez-site.xml # ========================================================================= : "${DB_DRIVER:=derby}" @@ -68,9 +77,30 @@ function initialize_hive { fi } +function append_java_opens { + local -n _opts=$1 + local opens=( + "--add-opens=java.base/java.lang=ALL-UNNAMED" + "--add-opens=java.base/java.util=ALL-UNNAMED" + "--add-opens=java.base/java.io=ALL-UNNAMED" + "--add-opens=java.base/java.net=ALL-UNNAMED" + "--add-opens=java.base/java.nio=ALL-UNNAMED" + "--add-opens=java.base/java.util.concurrent=ALL-UNNAMED" + "--add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED" + "--add-opens=java.base/java.util.regex=ALL-UNNAMED" + "--add-opens=java.base/java.lang.reflect=ALL-UNNAMED" + "--add-opens=java.sql/java.sql=ALL-UNNAMED" + "--add-opens=java.base/java.text=ALL-UNNAMED" + "-Dnet.bytebuddy.experimental=true" + ) + for opt in "${opens[@]}"; do + if [[ " ${_opts} " != *" ${opt} "* ]]; then + _opts="${_opts} ${opt}" + fi + done +} + function run_llap { - export HIVE_ZOOKEEPER_QUORUM="${HIVE_ZOOKEEPER_QUORUM:-zookeeper:2181}" - export HIVE_LLAP_DAEMON_SERVICE_HOSTS="${HIVE_LLAP_DAEMON_SERVICE_HOSTS:-@llap0}" export LLAP_MEMORY_MB="${LLAP_MEMORY_MB:-1024}" export LLAP_EXECUTORS="${LLAP_EXECUTORS:-1}" @@ -87,25 +117,8 @@ function run_llap { export LLAP_DAEMON_CONF_DIR="${LLAP_DAEMON_CONF_DIR:-$HIVE_CONF_DIR}" export LLAP_DAEMON_USER_CLASSPATH="${LLAP_DAEMON_USER_CLASSPATH:-$TEZ_HOME/*:$TEZ_HOME/lib/*:$HADOOP_HOME/share/hadoop/common/*:$HADOOP_HOME/share/hadoop/common/lib/*:$HADOOP_HOME/share/hadoop/yarn/*:$HADOOP_HOME/share/hadoop/yarn/lib/*:$HADOOP_HOME/share/hadoop/hdfs/*:$HADOOP_HOME/share/hadoop/hdfs/lib/*:$HADOOP_HOME/share/hadoop/mapreduce/*:$HADOOP_HOME/share/hadoop/mapreduce/lib/*:$HADOOP_HOME/share/hadoop/tools/lib/*}" - JAVA_ADD_OPENS=( - "--add-opens=java.base/java.lang=ALL-UNNAMED" - "--add-opens=java.base/java.util=ALL-UNNAMED" - "--add-opens=java.base/java.io=ALL-UNNAMED" - "--add-opens=java.base/java.net=ALL-UNNAMED" - "--add-opens=java.base/java.nio=ALL-UNNAMED" - "--add-opens=java.base/java.util.concurrent=ALL-UNNAMED" - "--add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED" - "--add-opens=java.base/java.util.regex=ALL-UNNAMED" - "--add-opens=java.base/java.lang.reflect=ALL-UNNAMED" - "--add-opens=java.sql/java.sql=ALL-UNNAMED" - "--add-opens=java.base/java.text=ALL-UNNAMED" - "-Dnet.bytebuddy.experimental=true" - ) - for opt in "${JAVA_ADD_OPENS[@]}"; do - if [[ " ${LLAP_DAEMON_OPTS:-} " != *" ${opt} "* ]]; then - LLAP_DAEMON_OPTS="${LLAP_DAEMON_OPTS:-} ${opt}" - fi - done + LLAP_DAEMON_OPTS="${LLAP_DAEMON_OPTS:-}" + append_java_opens LLAP_DAEMON_OPTS if [[ -n "${LLAP_EXTRA_OPTS:-}" ]]; then export LLAP_DAEMON_OPTS="${LLAP_DAEMON_OPTS:-} ${LLAP_EXTRA_OPTS}" @@ -121,6 +134,35 @@ function run_llap { exec "${LLAP_RUN_SCRIPT}" run "$@" } +function run_tezam { + : "${USER:=hive}" + : "${LOCAL_DIRS:="/tmp"}" + : "${LOG_DIRS:="/opt/tez/logs"}" + : "${APP_SUBMIT_TIME_ENV:=$(($(date +%s) * 1000))}" + : "${TEZ_AM_EXTERNAL_ID:="tez-session-${HOSTNAME:-tezam}"}" + export USER LOCAL_DIRS LOG_DIRS APP_SUBMIT_TIME_ENV TEZ_AM_EXTERNAL_ID + + export HADOOP_HOME="${HADOOP_HOME:-/opt/hadoop}" + export TEZ_HOME="${TEZ_HOME:-/opt/tez}" + export HIVE_HOME="${HIVE_HOME:-/opt/hive}" + export HADOOP_CONF_DIR="${HADOOP_CONF_DIR:-$HIVE_CONF_DIR}" + export TEZ_CONF_DIR="${TEZ_CONF_DIR:-$HADOOP_CONF_DIR}" + # service_plugins_descriptor.json references org.apache.hadoop.hive.llap.tezplugins.* (hive-llap-tez, etc.) + tezam_cp="${HADOOP_CONF_DIR}:${TEZ_CONF_DIR}:${TEZ_HOME}/*:${TEZ_HOME}/lib/*:${HIVE_HOME}/lib/*:${HADOOP_HOME}/share/hadoop/common/*:${HADOOP_HOME}/share/hadoop/common/lib/*:${HADOOP_HOME}/share/hadoop/yarn/*:${HADOOP_HOME}/share/hadoop/yarn/lib/*:${HADOOP_HOME}/share/hadoop/hdfs/*:${HADOOP_HOME}/share/hadoop/hdfs/lib/*:${HADOOP_HOME}/share/hadoop/mapreduce/*:${HADOOP_HOME}/share/hadoop/mapreduce/lib/*:${HADOOP_CLASSPATH:-}" + + local java_bin + local tezam_java_opts + java_bin="${JAVA_HOME:+$JAVA_HOME/bin/}java" + tezam_java_opts="${HADOOP_CLIENT_OPTS:-} -Dlog4j.configuration=file:${HIVE_CONF_DIR}/tez-log4j.properties" + append_java_opens tezam_java_opts + "${java_bin}" ${tezam_java_opts} -cp "${tezam_cp}" org.apache.tez.dag.app.DAGAppMaster --session "$@" + local rc=$? + if [[ ${rc} -ne 0 ]]; then + echo "DAGAppMaster exited with code ${rc}. See logs above for details." + fi + exit "${rc}" +} + export HIVE_CONF_DIR=$HIVE_HOME/conf if [ -d "${HIVE_CUSTOM_CONF_DIR:-}" ]; then find "${HIVE_CUSTOM_CONF_DIR}" -type f -exec \ @@ -129,7 +171,7 @@ if [ -d "${HIVE_CUSTOM_CONF_DIR:-}" ]; then export TEZ_CONF_DIR=$HIVE_CONF_DIR fi -export HADOOP_CLIENT_OPTS="$HADOOP_CLIENT_OPTS -Xmx1G $SERVICE_OPTS" +export HADOOP_CLIENT_OPTS="${HADOOP_CLIENT_OPTS:-} -Xmx1G ${SERVICE_OPTS:-}" if [[ "${SKIP_SCHEMA_INIT}" == "false" && ( "${SERVICE_NAME}" == "hiveserver2" || "${SERVICE_NAME}" == "metastore" ) ]]; then # handles schema initialization initialize_hive @@ -147,4 +189,6 @@ elif [ "${SERVICE_NAME}" == "metastore" ]; then fi elif [ "${SERVICE_NAME}" == "llap" ]; then run_llap "$@" +elif [ "${SERVICE_NAME}" == "tezam" ]; then + run_tezam "$@" fi