From 1cae19e1275ecf362f8c9ecb38fd7916f1bb6c5d Mon Sep 17 00:00:00 2001 From: Dongjoon Hyun Date: Tue, 9 Jun 2026 11:41:36 -0700 Subject: [PATCH] [SPARK-57351] Enable `spark.kubernetes.executor.useDriverPodIP` by default --- core/src/main/scala/org/apache/spark/SparkContext.scala | 2 +- docs/core-migration-guide.md | 4 ++++ docs/running-on-kubernetes.md | 2 +- .../src/main/scala/org/apache/spark/deploy/k8s/Config.scala | 2 +- 4 files changed, 7 insertions(+), 3 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/SparkContext.scala b/core/src/main/scala/org/apache/spark/SparkContext.scala index 0262144490ce8..e00e34b89b842 100644 --- a/core/src/main/scala/org/apache/spark/SparkContext.scala +++ b/core/src/main/scala/org/apache/spark/SparkContext.scala @@ -455,7 +455,7 @@ class SparkContext(config: SparkConf) extends Logging { // Set Spark driver host and port system properties. This explicitly sets the configuration // instead of relying on the default value of the config constant. if (SparkMasterRegex.isK8s(master) && - _conf.getBoolean("spark.kubernetes.executor.useDriverPodIP", false)) { + _conf.getBoolean("spark.kubernetes.executor.useDriverPodIP", true)) { logInfo("Use DRIVER_BIND_ADDRESS instead of DRIVER_HOST_ADDRESS as driver address " + "because spark.kubernetes.executor.useDriverPodIP is true in K8s mode.") _conf.set(DRIVER_HOST_ADDRESS, _conf.get(DRIVER_BIND_ADDRESS)) diff --git a/docs/core-migration-guide.md b/docs/core-migration-guide.md index 284cd7a8639b2..9be60572387ca 100644 --- a/docs/core-migration-guide.md +++ b/docs/core-migration-guide.md @@ -22,6 +22,10 @@ license: | * Table of contents {:toc} +## Upgrading from Core 4.2 to 4.3 + +- Since Spark 4.3, Spark executor pods connect to the driver via the driver pod IP directly instead of the driver's Kubernetes Service. To restore the legacy behavior, you can set `spark.kubernetes.executor.useDriverPodIP` to `false`. + ## Upgrading from Core 4.1 to 4.2 - Since Spark 4.2, Spark Master REST API uses Java 21 virtual threads by default when running on Java 21 or later. To restore the legacy behavior, you can set `spark.master.rest.virtualThread.enabled` to `false`. diff --git a/docs/running-on-kubernetes.md b/docs/running-on-kubernetes.md index aa753e259bcc7..9be54d8984869 100644 --- a/docs/running-on-kubernetes.md +++ b/docs/running-on-kubernetes.md @@ -1585,7 +1585,7 @@ See the [configuration page](configuration.html) for information on Spark config spark.kubernetes.executor.useDriverPodIP - false + true If true, executor pods use Driver pod IP directly instead of Driver Service. diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Config.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Config.scala index 0e79cd0d5606c..120881469e3a9 100644 --- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Config.scala +++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Config.scala @@ -125,7 +125,7 @@ private[spark] object Config extends Logging { .doc("If true, executor pods use Driver pod IP directly instead of Driver Service.") .version("4.1.0") .booleanConf - .createWithDefault(false) + .createWithDefault(true) val KUBERNETES_NAMESPACE = ConfigBuilder("spark.kubernetes.namespace")