Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line numberDiff line numberDiff line change
Expand Up@@ -257,7 +257,7 @@ protected void startRpcServer() {
int numHandlers =
HiveConf.getIntVar(conf, ConfVars.LLAP_TASK_COMMUNICATOR_LISTENER_THREAD_COUNT);
String[] portRange =
conf.get(HiveConf.ConfVars.LLAP_TASK_UMBILICAL_SERVER_PORT.varname)
HiveConf.getVar(conf, HiveConf.ConfVars.LLAP_TASK_UMBILICAL_SERVER_PORT)
.split("-");
boolean isHadoopSecurityAuthorizationEnabled = conf.getBoolean(
CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION, false);
Expand Down
1 change: 0 additions & 1 deletion packaging/src/docker/Dockerfile
Original file line numberDiff line numberDiff line change
Expand Up@@ -75,7 +75,6 @@ ARG UID=1000
ARG HADOOP_VERSION
ARG HIVE_VERSION
ARG TEZ_VERSION

# Install dependencies
RUN set -ex; \
microdnf update -y; \
Expand Down
23 changes: 17 additions & 6 deletions packaging/src/docker/build.sh
Original file line numberDiff line numberDiff line change
Expand Up@@ -22,11 +22,11 @@ HADOOP_VERSION=
TEZ_VERSION=
usage() {
cat <<EOF 1>&2
Usage: $0 [-h] [-hadoop <Hadoop version>] [-tez <Tez version>] [-hive <Hive version>] [-repo <Docker repo>]
Usage: $0 [-h] [-hadoop <Hadoop version>] -tez <Tez release version> [-hive <Hive version>] [-repo <Docker repo>]
Build the Hive Docker image (reused for LLAP too)
-help Display help
-hadoop Build image with the specified Hadoop version
-tez Build image with the specified Tez version
-hadoop Build image with the specified Hadoop version (default: from Maven pom)
-tez Required. Tez release tarball version (apache-tez-\$TEZ_VERSION-bin.tar.gz from archive)
-hive Build image with the specified Hive version
-repo Docker repository
EOF
Expand DownExpand Up@@ -64,6 +64,12 @@ while [ $# -gt 0 ]; do
esac
done

if [ -z "${TEZ_VERSION}" ]; then
echo "Error: -tez <Tez version> is required." >&2
usage
exit 1
fi

SCRIPT_DIR=$(cd $(dirname $0); pwd)
SOURCE_DIR=${SOURCE_DIR:-"$SCRIPT_DIR/../../.."}
repo=${REPO:-apache}
Expand DownExpand Up@@ -123,12 +129,17 @@ cp "$CACHE_DIR/apache-tez-$TEZ_VERSION-bin.tar.gz" "$WORK_DIR/"
cp -R "$SOURCE_DIR/packaging/src/docker/conf" "$WORK_DIR/"
cp -R "$SOURCE_DIR/packaging/src/docker/entrypoint.sh" "$WORK_DIR/"
cp "$SOURCE_DIR/packaging/src/docker/Dockerfile" "$WORK_DIR/"

DOCKER_BUILD_ARGS=(
--build-arg "HIVE_VERSION=$HIVE_VERSION"
--build-arg "HADOOP_VERSION=$HADOOP_VERSION"
--build-arg "TEZ_VERSION=$TEZ_VERSION"
)

docker build \
"$WORK_DIR" \
-f "$WORK_DIR/Dockerfile" \
-t "$repo/hive:$HIVE_VERSION" \
--build-arg "HIVE_VERSION=$HIVE_VERSION" \
--build-arg "HADOOP_VERSION=$HADOOP_VERSION" \
--build-arg "TEZ_VERSION=$TEZ_VERSION"
"${DOCKER_BUILD_ARGS[@]}"

rm -r "${WORK_DIR}"
21 changes: 21 additions & 0 deletions packaging/src/docker/conf/hive-site.xml.template
Original file line numberDiff line numberDiff line change
Expand Up@@ -88,4 +88,25 @@
<name>hive.query.results.cache.directory</name>
<value>${HIVE_QUERY_RESULTS_CACHE_DIRECTORY}</value>
</property>
<property>
<name>hive.server2.tez.initialize.default.sessions</name>
<value>false</value>

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

why not initialize ?

Copy link
Copy Markdown
ContributorAuthor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

it's a yarn-based thing that HS2 initializes a few sessions on startup
in the standalone mode, HS2 has no control over them, just discovers them via ZK

</property>
<property>
<name>hive.server2.tez.use.external.sessions</name>
<value>${HIVE_SERVER2_TEZ_USE_EXTERNAL_SESSIONS}</value>
</property>
<!--
A registry namespace prefix is a hardcoded prefix for Tez external sessions.
The actual tez.am.registry.namespace value is appended to this prefix.
Once hive can use the registry client that Tez provides (ZkAMRegistryClient), this property will be removed.

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

is there a ticket for that? needs Tez upgrade?

@abstractdogabstractdogApr 17, 2026

Copy link
Copy Markdown
ContributorAuthor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

-->
<property>
<name>hive.server2.tez.external.sessions.namespace</name>
<value>/tez-external-sessions${TEZ_AM_REGISTRY_NAMESPACE}</value>
</property>
<property>
<name>hive.server2.tez.external.sessions.registry.class</name>
<value>org.apache.hadoop.hive.ql.exec.tez.ZookeeperExternalSessionsRegistryClient</value>
</property>
</configuration>
22 changes: 22 additions & 0 deletions packaging/src/docker/conf/service_plugins_descriptor.json
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
{
"taskSchedulerDescriptors": [
{
"className": "org.apache.hadoop.hive.llap.tezplugins.LlapTaskSchedulerService",
"entityName": "LLAP"
}
],
"containerLauncherDescriptors": [
{
"className": "org.apache.hadoop.hive.llap.tezplugins.LlapContainerLauncher",
"entityName": "LLAP"
}
],
"taskCommunicatorDescriptors": [
{
"className": "org.apache.hadoop.hive.llap.tezplugins.LlapTaskCommunicator",
"entityName": "LLAP"
}
],
"enableContainers": false,
"enableUber": false
}
5 changes: 5 additions & 0 deletions packaging/src/docker/conf/tez-log4j.properties
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
log4j.rootLogger=INFO, console

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

maybe, simply [tez-log4j.properties]

log4j.appender.console=org.apache.log4j.ConsoleAppender
log4j.appender.console.Target=System.err
log4j.appender.console.layout=org.apache.log4j.PatternLayout
log4j.appender.console.layout.ConversionPattern=%d{ISO8601} %-5p %c{1} - %m%n
74 changes: 74 additions & 0 deletions packaging/src/docker/conf/tez-site.xml.template
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,74 @@
<?xml version="1.0"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at

http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<configuration>
<property>
<name>tez.am.mode.session</name>
<value>true</value>
</property>
<property>
<name>tez.am.framework.mode</name>
<value>${TEZ_FRAMEWORK_MODE}</value>
</property>
<property>
<name>tez.am.zookeeper.quorum</name>
<value>${TEZ_AM_ZOOKEEPER_QUORUM}</value>
</property>
<property>
<name>tez.am.registry.namespace</name>
<value>${TEZ_AM_REGISTRY_NAMESPACE}</value>
</property>
<property>
<name>tez.local.mode</name>
<value>false</value>
</property>
<property>
<name>tez.am.tez-ui.webservice.enable</name>
<value>false</value>
</property>
<!-- Tez AM should not timeout in ZK Mode -->
<property>
<name>tez.session.am.dag.submit.timeout.secs</name>
<value>-1</value>
</property>
<property>
<name>tez.ignore.lib.uris</name>
<value>true</value>
</property>
<property>
<name>tez.am.disable.client-version-check</name>
<value>true</value>
</property>

<!--
In standalone AM mode with LLAP, the task scheduler (LlapTaskSchedulerService) and task
communicator (LlapTaskCommunicator) are instantiated at AM startup from service_plugins_descriptor.json
on the classpath — not from the DAG payload as in YARN-submitted mode. These plugins read their
configuration from TezConfiguration, which is built exclusively from tez-site.xml on the classpath.
Hive-specific properties must therefore appear here so the AM can find LLAP daemons and connect
to them, even though they are conceptually Hive settings.
-->
<property>
<name>hive.zookeeper.quorum</name>
<value>${HIVE_ZOOKEEPER_QUORUM}</value>
</property>
<property>
<name>hive.llap.daemon.service.hosts</name>
<value>${HIVE_LLAP_DAEMON_SERVICE_HOSTS}</value>
</property>
</configuration>
37 changes: 29 additions & 8 deletions packaging/src/docker/docker-compose.yml
Original file line numberDiff line numberDiff line change
Expand Up@@ -132,14 +132,35 @@ services:
- zookeeper_datalog:/datalog
- zookeeper_logs:/logs

#TODO Tez AM container (in the meantime, the HS2(with local Tez AM) + LLAP daemon setup is working properly)
# 1. Define and use a Tez AM image from HIVE-29419 or TEZ-4682
# 2. Configure TezAM to use Zookeeper Llap Registry to discover the LLAP daemon
# 3. Configure HiveServer2 to use the Tez AM Zookeeper Registry to discover the Tez AM
# Prerequisites:
# - tez-api 1.0.0-SNAPSHOT jar injected into HiveSever2 until Tez 1.0.0 is released
# - make HIVE-29477 happen to let HiveServer2 use Tez external sessions
# 4. Define hadoop components here to be used by all the containers (working example can be found at TEZ-4682), currently a local volume
tezam:
profiles:
- llap

@deniskuzZdeniskuzZApr 23, 2026

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

should we add standalone support later? without llap daemons -> Tez containers?

@abstractdogabstractdogApr 23, 2026

Copy link
Copy Markdown
ContributorAuthor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

what would this standalone mode look like? you mean: hs2 + tezam + tezchild (classic single task executor)?

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

yea, something like that. not useful?

Copy link
Copy Markdown
ContributorAuthor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

sorry, forgot to respond: short term, it's not worth the effort from Hive project's point of view, given that we adapted at least one of our main execution modes (LLAP vs. Tez container mode), LLAP, to cloud, but long term, it's a good-to-have and not only for Hive, but Tez also, that's what TEZ-4662 is about

image: apache/hive:${HIVE_VERSION}

@deniskuzZdeniskuzZApr 17, 2026

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

is it Ok to reuse hive image for TezAM? IDK just asking. is is the same downstream?

@abstractdogabstractdogApr 17, 2026

Copy link
Copy Markdown
ContributorAuthor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think it's ok to reuse, actually, we build both (Hive specific TezAM image + Tez AM image in Tez project), see the motivation on HIVE-29419
downstream we have only a hive image for TezAM, but upstream, the Tez project needs to have its own, TEZ-4682, most probably mimics the Tez Container mode (DAGAppMaster + TezChiild), not the LLAP

container_name: tezam
hostname: tezam
depends_on:
- zookeeper
restart: on-failure:3
environment:
USER: hive
SERVICE_NAME: 'tezam'

TEZ_FRAMEWORK_MODE: STANDALONE_ZOOKEEPER
TEZ_AM_ZOOKEEPER_QUORUM: zookeeper:2181

# LLAP daemon discovery
HIVE_ZOOKEEPER_QUORUM: zookeeper:2181
HIVE_LLAP_DAEMON_SERVICE_HOSTS: '@llap0'

# Directories shared between HiveServer2 and LLAP daemon
HIVE_SCRATCH_DIR: /opt/hive/scratch
HIVE_QUERY_RESULTS_CACHE_DIRECTORY: /opt/hive/scratch/_resultscache_

volumes:
- warehouse:/opt/hive/data/warehouse
- scratch:/opt/hive/scratch
networks:
- hive

llapdaemon:
profiles:
Expand Down
88 changes: 66 additions & 22 deletions packaging/src/docker/entrypoint.sh
Original file line numberDiff line numberDiff line change
Expand Up@@ -41,8 +41,17 @@ export HIVE_WAREHOUSE_PATH="${HIVE_WAREHOUSE_PATH:-/opt/hive/data/warehouse}"
export HIVE_SCRATCH_DIR="${HIVE_SCRATCH_DIR:-/opt/hive/scratch}"
export HIVE_QUERY_RESULTS_CACHE_DIRECTORY="${HIVE_WAREHOUSE_PATH:-/opt/hive/scratch/_resultscache_}"

export HIVE_ZOOKEEPER_QUORUM="${HIVE_ZOOKEEPER_QUORUM:-zookeeper:2181}"
export HIVE_LLAP_DAEMON_SERVICE_HOSTS="${HIVE_LLAP_DAEMON_SERVICE_HOSTS:-@llap0}"

export HIVE_SERVER2_TEZ_USE_EXTERNAL_SESSIONS="${HIVE_SERVER2_TEZ_USE_EXTERNAL_SESSIONS:-true}"
export TEZ_FRAMEWORK_MODE="${TEZ_FRAMEWORK_MODE:-STANDALONE_ZOOKEEPER}"
export TEZ_AM_REGISTRY_NAMESPACE="${TEZ_AM_REGISTRY_NAMESPACE:-/tez_am/server}"
export TEZ_AM_ZOOKEEPER_QUORUM="${TEZ_AM_ZOOKEEPER_QUORUM:-${HIVE_ZOOKEEPER_QUORUM}}"

envsubst < $HIVE_HOME/conf/core-site.xml.template > $HIVE_HOME/conf/core-site.xml
envsubst < $HIVE_HOME/conf/hive-site.xml.template > $HIVE_HOME/conf/hive-site.xml
envsubst < $HIVE_HOME/conf/tez-site.xml.template > $HIVE_HOME/conf/tez-site.xml
# =========================================================================

: "${DB_DRIVER:=derby}"
Expand All@@ -68,9 +77,30 @@ function initialize_hive {
fi
}

function append_java_opens {
local -n _opts=$1
local opens=(
"--add-opens=java.base/java.lang=ALL-UNNAMED"
"--add-opens=java.base/java.util=ALL-UNNAMED"
"--add-opens=java.base/java.io=ALL-UNNAMED"
"--add-opens=java.base/java.net=ALL-UNNAMED"
"--add-opens=java.base/java.nio=ALL-UNNAMED"
"--add-opens=java.base/java.util.concurrent=ALL-UNNAMED"
"--add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED"
"--add-opens=java.base/java.util.regex=ALL-UNNAMED"
"--add-opens=java.base/java.lang.reflect=ALL-UNNAMED"
"--add-opens=java.sql/java.sql=ALL-UNNAMED"
"--add-opens=java.base/java.text=ALL-UNNAMED"
"-Dnet.bytebuddy.experimental=true"
)
for opt in "${opens[@]}"; do
if [[ " ${_opts} " != *" ${opt} "* ]]; then
_opts="${_opts} ${opt}"
fi
done
}

function run_llap {
export HIVE_ZOOKEEPER_QUORUM="${HIVE_ZOOKEEPER_QUORUM:-zookeeper:2181}"
export HIVE_LLAP_DAEMON_SERVICE_HOSTS="${HIVE_LLAP_DAEMON_SERVICE_HOSTS:-@llap0}"
export LLAP_MEMORY_MB="${LLAP_MEMORY_MB:-1024}"
export LLAP_EXECUTORS="${LLAP_EXECUTORS:-1}"

Expand All@@ -87,25 +117,8 @@ function run_llap {
export LLAP_DAEMON_CONF_DIR="${LLAP_DAEMON_CONF_DIR:-$HIVE_CONF_DIR}"
export LLAP_DAEMON_USER_CLASSPATH="${LLAP_DAEMON_USER_CLASSPATH:-$TEZ_HOME/*:$TEZ_HOME/lib/*:$HADOOP_HOME/share/hadoop/common/*:$HADOOP_HOME/share/hadoop/common/lib/*:$HADOOP_HOME/share/hadoop/yarn/*:$HADOOP_HOME/share/hadoop/yarn/lib/*:$HADOOP_HOME/share/hadoop/hdfs/*:$HADOOP_HOME/share/hadoop/hdfs/lib/*:$HADOOP_HOME/share/hadoop/mapreduce/*:$HADOOP_HOME/share/hadoop/mapreduce/lib/*:$HADOOP_HOME/share/hadoop/tools/lib/*}"

JAVA_ADD_OPENS=(
"--add-opens=java.base/java.lang=ALL-UNNAMED"
"--add-opens=java.base/java.util=ALL-UNNAMED"
"--add-opens=java.base/java.io=ALL-UNNAMED"
"--add-opens=java.base/java.net=ALL-UNNAMED"
"--add-opens=java.base/java.nio=ALL-UNNAMED"
"--add-opens=java.base/java.util.concurrent=ALL-UNNAMED"
"--add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED"
"--add-opens=java.base/java.util.regex=ALL-UNNAMED"
"--add-opens=java.base/java.lang.reflect=ALL-UNNAMED"
"--add-opens=java.sql/java.sql=ALL-UNNAMED"
"--add-opens=java.base/java.text=ALL-UNNAMED"
"-Dnet.bytebuddy.experimental=true"
)
for opt in "${JAVA_ADD_OPENS[@]}"; do
if [[ " ${LLAP_DAEMON_OPTS:-} " != *" ${opt} "* ]]; then
LLAP_DAEMON_OPTS="${LLAP_DAEMON_OPTS:-} ${opt}"
fi
done
LLAP_DAEMON_OPTS="${LLAP_DAEMON_OPTS:-}"
append_java_opens LLAP_DAEMON_OPTS

if [[ -n "${LLAP_EXTRA_OPTS:-}" ]]; then
export LLAP_DAEMON_OPTS="${LLAP_DAEMON_OPTS:-} ${LLAP_EXTRA_OPTS}"
Expand All@@ -121,6 +134,35 @@ function run_llap {
exec "${LLAP_RUN_SCRIPT}" run "$@"
}

function run_tezam {
: "${USER:=hive}"
: "${LOCAL_DIRS:="/tmp"}"
: "${LOG_DIRS:="/opt/tez/logs"}"
: "${APP_SUBMIT_TIME_ENV:=$(($(date +%s) * 1000))}"
: "${TEZ_AM_EXTERNAL_ID:="tez-session-${HOSTNAME:-tezam}"}"
export USER LOCAL_DIRS LOG_DIRS APP_SUBMIT_TIME_ENV TEZ_AM_EXTERNAL_ID

export HADOOP_HOME="${HADOOP_HOME:-/opt/hadoop}"
export TEZ_HOME="${TEZ_HOME:-/opt/tez}"
export HIVE_HOME="${HIVE_HOME:-/opt/hive}"
export HADOOP_CONF_DIR="${HADOOP_CONF_DIR:-$HIVE_CONF_DIR}"
export TEZ_CONF_DIR="${TEZ_CONF_DIR:-$HADOOP_CONF_DIR}"
# service_plugins_descriptor.json references org.apache.hadoop.hive.llap.tezplugins.* (hive-llap-tez, etc.)
tezam_cp="${HADOOP_CONF_DIR}:${TEZ_CONF_DIR}:${TEZ_HOME}/*:${TEZ_HOME}/lib/*:${HIVE_HOME}/lib/*:${HADOOP_HOME}/share/hadoop/common/*:${HADOOP_HOME}/share/hadoop/common/lib/*:${HADOOP_HOME}/share/hadoop/yarn/*:${HADOOP_HOME}/share/hadoop/yarn/lib/*:${HADOOP_HOME}/share/hadoop/hdfs/*:${HADOOP_HOME}/share/hadoop/hdfs/lib/*:${HADOOP_HOME}/share/hadoop/mapreduce/*:${HADOOP_HOME}/share/hadoop/mapreduce/lib/*:${HADOOP_CLASSPATH:-}"

local java_bin
local tezam_java_opts
java_bin="${JAVA_HOME:+$JAVA_HOME/bin/}java"
tezam_java_opts="${HADOOP_CLIENT_OPTS:-} -Dlog4j.configuration=file:${HIVE_CONF_DIR}/tez-log4j.properties"
append_java_opens tezam_java_opts
"${java_bin}" ${tezam_java_opts} -cp "${tezam_cp}" org.apache.tez.dag.app.DAGAppMaster --session "$@"
local rc=$?
if [[ ${rc} -ne 0 ]]; then
echo "DAGAppMaster exited with code ${rc}. See logs above for details."
fi
exit "${rc}"
}

export HIVE_CONF_DIR=$HIVE_HOME/conf
if [ -d "${HIVE_CUSTOM_CONF_DIR:-}" ]; then
find "${HIVE_CUSTOM_CONF_DIR}" -type f -exec \
Expand All@@ -129,7 +171,7 @@ if [ -d "${HIVE_CUSTOM_CONF_DIR:-}" ]; then
export TEZ_CONF_DIR=$HIVE_CONF_DIR
fi

export HADOOP_CLIENT_OPTS="$HADOOP_CLIENT_OPTS -Xmx1G $SERVICE_OPTS"
export HADOOP_CLIENT_OPTS="${HADOOP_CLIENT_OPTS:-} -Xmx1G ${SERVICE_OPTS:-}"
if [[ "${SKIP_SCHEMA_INIT}" == "false" && ( "${SERVICE_NAME}" == "hiveserver2" || "${SERVICE_NAME}" == "metastore" ) ]]; then
# handles schema initialization
initialize_hive
Expand All@@ -147,4 +189,6 @@ elif [ "${SERVICE_NAME}" == "metastore" ]; then
fi
elif [ "${SERVICE_NAME}" == "llap" ]; then
run_llap "$@"
elif [ "${SERVICE_NAME}" == "tezam" ]; then
run_tezam "$@"
fi
Loading