From e07c230afd6cfc378707799a3a942a1c84343dfd Mon Sep 17 00:00:00 2001 From: gwang3 Date: Fri, 20 Sep 2019 20:52:24 +0800 Subject: [PATCH 1/9] [SPARK-29189] Add an option to ignore block locations when listing file (cherry picked from commit cdef51c166fbbb1321231bbfd6a7359ccbb3109c) --- .../scala/org/apache/spark/sql/internal/SQLConf.scala | 10 ++++++++++ .../sql/execution/datasources/InMemoryFileIndex.scala | 7 +++++-- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala index 0ec661fc16c88..e25688a9c5173 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala @@ -1979,6 +1979,14 @@ object SQLConf { .doc("When true, the ArrayExists will follow the three-valued boolean logic.") .booleanConf .createWithDefault(true) + + val IGNORE_DATA_LOCALITY = + buildConf("spark.sql.ignore.datalocality") + .doc("If it is set to true, Spark won't fetch the block locations for each file on " + + "listing files, which will greatly accelerate the list file operation, while loss " + + "data locality.") + .booleanConf + .createWithDefault(false) } /** @@ -2475,6 +2483,8 @@ class SQLConf extends Serializable with Logging { def defaultV2Catalog: Option[String] = getConf(DEFAULT_V2_CATALOG) + def ignoreDataLocality: Boolean = getConf(SQLConf.IGNORE_DATA_LOCALITY) + /** ********************** SQLConf functionality methods ************ */ /** Set Spark SQL configuration properties. */ diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/InMemoryFileIndex.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/InMemoryFileIndex.scala index cf7a13050f66c..9d3df82bf80ef 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/InMemoryFileIndex.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/InMemoryFileIndex.scala @@ -290,6 +290,7 @@ object InMemoryFileIndex extends Logging { isRootPath: Boolean): Seq[FileStatus] = { logTrace(s"Listing $path") val fs = path.getFileSystem(hadoopConf) + val ignoreLocality = sessionOpt.map(_.sqlContext.conf.ignoreDataLocality).getOrElse(false) // Note that statuses only include FileStatus for the files and dirs directly under path, // and does not include anything else recursively. @@ -299,7 +300,7 @@ object InMemoryFileIndex extends Logging { // to retrieve the file status with the file block location. The reason to still fallback // to listStatus is because the default implementation would potentially throw a // FileNotFoundException which is better handled by doing the lookups manually below. - case _: DistributedFileSystem => + case _: DistributedFileSystem if !ignoreLocality => val remoteIter = fs.listLocatedStatus(path) new Iterator[LocatedFileStatus]() { def next(): LocatedFileStatus = remoteIter.next @@ -376,7 +377,7 @@ object InMemoryFileIndex extends Logging { // - Here we are calling `getFileBlockLocations` in a sequential manner, but it should not // be a big deal since we always use to `bulkListLeafFiles` when the number of // paths exceeds threshold. - case f => + case f if !ignoreLocality => // The other constructor of LocatedFileStatus will call FileStatus.getPermission(), // which is very slow on some file system (RawLocalFileSystem, which is launch a // subprocess and parse the stdout). @@ -400,6 +401,8 @@ object InMemoryFileIndex extends Logging { missingFiles += f.getPath.toString None } + + case f => Some(f) } if (missingFiles.nonEmpty) { From b7f9f0359da0eca1840cebbdc148ef743c9c3c77 Mon Sep 17 00:00:00 2001 From: gwang3 Date: Sun, 22 Sep 2019 10:50:45 +0800 Subject: [PATCH 2/9] add ut --- .../datasources/FileIndexSuite.scala | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/FileIndexSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/FileIndexSuite.scala index 4b086e830e456..d3b15e736ffdd 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/FileIndexSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/FileIndexSuite.scala @@ -416,6 +416,37 @@ class FileIndexSuite extends SharedSparkSession { } } + test("SPARK-29189: Add an option to ignore block locations when listing file") { + withTempDir { dir => + val partitionDirectory = new File(dir, "a=foo") + partitionDirectory.mkdir() + for (i <- 1 to 8) { + val file = new File(partitionDirectory, i + ".txt") + stringToFile(file, "text") + } + val path = new Path(dir.getCanonicalPath) + + var withBlockLocations, withoutBlockLocations = Seq.empty[FileStatus] + withSQLConf(SQLConf.IGNORE_DATA_LOCALITY.key -> "false", + "fs.file.impl" -> classOf[SpecialBlockLocationFileSystem].getName) { + val fileIndex = new InMemoryFileIndex(spark, Seq(path), Map.empty, None) + withBlockLocations = fileIndex. + listLeafFiles(Seq(new Path(partitionDirectory.getPath))).toSeq + } + withSQLConf(SQLConf.IGNORE_DATA_LOCALITY.key -> "true", + "fs.file.impl" -> classOf[SpecialBlockLocationFileSystem].getName) { + val fileIndex = new InMemoryFileIndex(spark, Seq(path), Map.empty, None) + withoutBlockLocations = fileIndex. + listLeafFiles(Seq(new Path(partitionDirectory.getPath))).toSeq + } + assert(withBlockLocations.size == withoutBlockLocations.size) + assert(withBlockLocations.forall(b => b.isInstanceOf[LocatedFileStatus] && + b.asInstanceOf[LocatedFileStatus].getBlockLocations.nonEmpty)) + assert(withBlockLocations.forall(b => b.isInstanceOf[LocatedFileStatus] && + b.asInstanceOf[LocatedFileStatus].getBlockLocations.isEmpty)) + assert(withoutBlockLocations.forall(withBlockLocations.contains)) + } + } } object DeletionRaceFileSystem { From 0da290368ae7bf59aa37b3a1aac514808510f484 Mon Sep 17 00:00:00 2001 From: gwang3 Date: Sun, 22 Sep 2019 15:57:49 +0800 Subject: [PATCH 3/9] fix ut --- .../spark/sql/execution/datasources/InMemoryFileIndex.scala | 3 ++- .../spark/sql/execution/datasources/FileIndexSuite.scala | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/InMemoryFileIndex.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/InMemoryFileIndex.scala index 9d3df82bf80ef..3c6046e49a815 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/InMemoryFileIndex.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/InMemoryFileIndex.scala @@ -31,6 +31,7 @@ import org.apache.spark.internal.Logging import org.apache.spark.metrics.source.HiveCatalogMetrics import org.apache.spark.sql.SparkSession import org.apache.spark.sql.execution.streaming.FileStreamSink +import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types.StructType import org.apache.spark.util.SerializableConfiguration @@ -290,7 +291,7 @@ object InMemoryFileIndex extends Logging { isRootPath: Boolean): Seq[FileStatus] = { logTrace(s"Listing $path") val fs = path.getFileSystem(hadoopConf) - val ignoreLocality = sessionOpt.map(_.sqlContext.conf.ignoreDataLocality).getOrElse(false) + val ignoreLocality = hadoopConf.getBoolean(SQLConf.IGNORE_DATA_LOCALITY.key, false) // Note that statuses only include FileStatus for the files and dirs directly under path, // and does not include anything else recursively. diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/FileIndexSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/FileIndexSuite.scala index d3b15e736ffdd..9c5ab5c698380 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/FileIndexSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/FileIndexSuite.scala @@ -442,8 +442,8 @@ class FileIndexSuite extends SharedSparkSession { assert(withBlockLocations.size == withoutBlockLocations.size) assert(withBlockLocations.forall(b => b.isInstanceOf[LocatedFileStatus] && b.asInstanceOf[LocatedFileStatus].getBlockLocations.nonEmpty)) - assert(withBlockLocations.forall(b => b.isInstanceOf[LocatedFileStatus] && - b.asInstanceOf[LocatedFileStatus].getBlockLocations.isEmpty)) + assert(withoutBlockLocations.forall(b => b.isInstanceOf[FileStatus] && + !b.isInstanceOf[LocatedFileStatus])) assert(withoutBlockLocations.forall(withBlockLocations.contains)) } } From a659e6762c3902f5dec9beaefb349839340f1bfd Mon Sep 17 00:00:00 2001 From: gwang3 Date: Mon, 23 Sep 2019 14:47:10 +0800 Subject: [PATCH 4/9] tiny fix --- .../org/apache/spark/sql/internal/SQLConf.scala | 16 ++++++++-------- .../datasources/InMemoryFileIndex.scala | 6 +++++- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala index e25688a9c5173..904325ed3399c 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala @@ -832,6 +832,14 @@ object SQLConf { .intConf .createWithDefault(10000) + val IGNORE_DATA_LOCALITY = + buildConf("spark.sql.sources.ignore.datalocality") + .doc("If it is set to true, Spark will not fetch the block locations for each file on " + + "listing files, which will greatly accelerate the list file operation, while loss " + + "data locality.") + .booleanConf + .createWithDefault(false) + // Whether to automatically resolve ambiguity in join conditions for self-joins. // See SPARK-6231. val DATAFRAME_SELF_JOIN_AUTO_RESOLVE_AMBIGUITY = @@ -1979,14 +1987,6 @@ object SQLConf { .doc("When true, the ArrayExists will follow the three-valued boolean logic.") .booleanConf .createWithDefault(true) - - val IGNORE_DATA_LOCALITY = - buildConf("spark.sql.ignore.datalocality") - .doc("If it is set to true, Spark won't fetch the block locations for each file on " + - "listing files, which will greatly accelerate the list file operation, while loss " + - "data locality.") - .booleanConf - .createWithDefault(false) } /** diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/InMemoryFileIndex.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/InMemoryFileIndex.scala index 3c6046e49a815..2aa235708ed01 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/InMemoryFileIndex.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/InMemoryFileIndex.scala @@ -172,6 +172,7 @@ object InMemoryFileIndex extends Logging { areRootPaths: Boolean): Seq[(Path, Seq[FileStatus])] = { val ignoreMissingFiles = sparkSession.sessionState.conf.ignoreMissingFiles + val ignoreLocality = sparkSession.sessionState.conf.ignoreDataLocality // Short-circuits parallel listing when serial listing is likely to be faster. if (paths.size <= sparkSession.sessionState.conf.parallelPartitionDiscoveryThreshold) { @@ -182,6 +183,7 @@ object InMemoryFileIndex extends Logging { filter, Some(sparkSession), ignoreMissingFiles = ignoreMissingFiles, + ignoreLocality = ignoreLocality, isRootPath = areRootPaths) (path, leafFiles) } @@ -222,6 +224,7 @@ object InMemoryFileIndex extends Logging { filter, None, ignoreMissingFiles = ignoreMissingFiles, + ignoreLocality = ignoreLocality, isRootPath = areRootPaths) (path, leafFiles) }.iterator @@ -288,10 +291,10 @@ object InMemoryFileIndex extends Logging { filter: PathFilter, sessionOpt: Option[SparkSession], ignoreMissingFiles: Boolean, + ignoreLocality: Boolean, isRootPath: Boolean): Seq[FileStatus] = { logTrace(s"Listing $path") val fs = path.getFileSystem(hadoopConf) - val ignoreLocality = hadoopConf.getBoolean(SQLConf.IGNORE_DATA_LOCALITY.key, false) // Note that statuses only include FileStatus for the files and dirs directly under path, // and does not include anything else recursively. @@ -355,6 +358,7 @@ object InMemoryFileIndex extends Logging { filter, sessionOpt, ignoreMissingFiles = ignoreMissingFiles, + ignoreLocality = ignoreLocality, isRootPath = false) } } From 22dacc6c6eea8bc739084d40faa320a6f9ceea54 Mon Sep 17 00:00:00 2001 From: gwang3 Date: Tue, 24 Sep 2019 10:23:47 +0800 Subject: [PATCH 5/9] Trivial fix --- .../apache/spark/sql/internal/SQLConf.scala | 3 +- .../datasources/FileIndexSuite.scala | 33 ++++++++++--------- 2 files changed, 19 insertions(+), 17 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala index 904325ed3399c..e4c54d8e2998b 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala @@ -835,8 +835,9 @@ object SQLConf { val IGNORE_DATA_LOCALITY = buildConf("spark.sql.sources.ignore.datalocality") .doc("If it is set to true, Spark will not fetch the block locations for each file on " + - "listing files, which will greatly accelerate the list file operation, while loss " + + "listing files, which will boost listing file operation, at the cost of loss of " + "data locality.") + .internal() .booleanConf .createWithDefault(false) diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/FileIndexSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/FileIndexSuite.scala index 9c5ab5c698380..73a77b9810384 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/FileIndexSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/FileIndexSuite.scala @@ -416,7 +416,7 @@ class FileIndexSuite extends SharedSparkSession { } } - test("SPARK-29189: Add an option to ignore block locations when listing file") { + test("Add an option to ignore block locations when listing file") { withTempDir { dir => val partitionDirectory = new File(dir, "a=foo") partitionDirectory.mkdir() @@ -426,25 +426,26 @@ class FileIndexSuite extends SharedSparkSession { } val path = new Path(dir.getCanonicalPath) - var withBlockLocations, withoutBlockLocations = Seq.empty[FileStatus] withSQLConf(SQLConf.IGNORE_DATA_LOCALITY.key -> "false", "fs.file.impl" -> classOf[SpecialBlockLocationFileSystem].getName) { val fileIndex = new InMemoryFileIndex(spark, Seq(path), Map.empty, None) - withBlockLocations = fileIndex. - listLeafFiles(Seq(new Path(partitionDirectory.getPath))).toSeq - } - withSQLConf(SQLConf.IGNORE_DATA_LOCALITY.key -> "true", - "fs.file.impl" -> classOf[SpecialBlockLocationFileSystem].getName) { - val fileIndex = new InMemoryFileIndex(spark, Seq(path), Map.empty, None) - withoutBlockLocations = fileIndex. - listLeafFiles(Seq(new Path(partitionDirectory.getPath))).toSeq + val withBlockLocations = fileIndex. + listLeafFiles(Seq(new Path(partitionDirectory.getPath))) + + withSQLConf(SQLConf.IGNORE_DATA_LOCALITY.key -> "true", + "fs.file.impl" -> classOf[SpecialBlockLocationFileSystem].getName) { + val fileIndex = new InMemoryFileIndex(spark, Seq(path), Map.empty, None) + val withoutBlockLocations = fileIndex. + listLeafFiles(Seq(new Path(partitionDirectory.getPath))) + + assert(withBlockLocations.size == withoutBlockLocations.size) + assert(withBlockLocations.forall(b => b.isInstanceOf[LocatedFileStatus] && + b.asInstanceOf[LocatedFileStatus].getBlockLocations.nonEmpty)) + assert(withoutBlockLocations.forall(b => b.isInstanceOf[FileStatus] && + !b.isInstanceOf[LocatedFileStatus])) + assert(withoutBlockLocations.forall(withBlockLocations.contains)) + } } - assert(withBlockLocations.size == withoutBlockLocations.size) - assert(withBlockLocations.forall(b => b.isInstanceOf[LocatedFileStatus] && - b.asInstanceOf[LocatedFileStatus].getBlockLocations.nonEmpty)) - assert(withoutBlockLocations.forall(b => b.isInstanceOf[FileStatus] && - !b.isInstanceOf[LocatedFileStatus])) - assert(withoutBlockLocations.forall(withBlockLocations.contains)) } } } From eb1a802106e1ed9afe9049ec061d105b6812b37e Mon Sep 17 00:00:00 2001 From: gwang3 Date: Wed, 25 Sep 2019 10:16:39 +0800 Subject: [PATCH 6/9] fix code style --- .../apache/spark/sql/execution/datasources/FileIndexSuite.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/FileIndexSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/FileIndexSuite.scala index 73a77b9810384..d087ea47f68ee 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/FileIndexSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/FileIndexSuite.scala @@ -433,7 +433,7 @@ class FileIndexSuite extends SharedSparkSession { listLeafFiles(Seq(new Path(partitionDirectory.getPath))) withSQLConf(SQLConf.IGNORE_DATA_LOCALITY.key -> "true", - "fs.file.impl" -> classOf[SpecialBlockLocationFileSystem].getName) { + "fs.file.impl" -> classOf[SpecialBlockLocationFileSystem].getName) { val fileIndex = new InMemoryFileIndex(spark, Seq(path), Map.empty, None) val withoutBlockLocations = fileIndex. listLeafFiles(Seq(new Path(partitionDirectory.getPath))) From 475abba5ee0da99741724d8783b9de27e85ef852 Mon Sep 17 00:00:00 2001 From: gwang3 Date: Thu, 26 Sep 2019 22:13:15 +0800 Subject: [PATCH 7/9] Trivial fix --- .../spark/sql/execution/datasources/FileIndexSuite.scala | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/FileIndexSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/FileIndexSuite.scala index d087ea47f68ee..a7a2349a1dfb9 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/FileIndexSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/FileIndexSuite.scala @@ -425,16 +425,13 @@ class FileIndexSuite extends SharedSparkSession { stringToFile(file, "text") } val path = new Path(dir.getCanonicalPath) - + val fileIndex = new InMemoryFileIndex(spark, Seq(path), Map.empty, None) withSQLConf(SQLConf.IGNORE_DATA_LOCALITY.key -> "false", "fs.file.impl" -> classOf[SpecialBlockLocationFileSystem].getName) { - val fileIndex = new InMemoryFileIndex(spark, Seq(path), Map.empty, None) val withBlockLocations = fileIndex. listLeafFiles(Seq(new Path(partitionDirectory.getPath))) - withSQLConf(SQLConf.IGNORE_DATA_LOCALITY.key -> "true", - "fs.file.impl" -> classOf[SpecialBlockLocationFileSystem].getName) { - val fileIndex = new InMemoryFileIndex(spark, Seq(path), Map.empty, None) + withSQLConf(SQLConf.IGNORE_DATA_LOCALITY.key -> "true") { val withoutBlockLocations = fileIndex. listLeafFiles(Seq(new Path(partitionDirectory.getPath))) From e500bcd8b375a61ad9410511db5bcf4d92c79a9f Mon Sep 17 00:00:00 2001 From: gwang3 Date: Sun, 29 Sep 2019 11:44:29 +0800 Subject: [PATCH 8/9] remove unused import --- .../spark/sql/execution/datasources/InMemoryFileIndex.scala | 1 - 1 file changed, 1 deletion(-) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/InMemoryFileIndex.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/InMemoryFileIndex.scala index 2aa235708ed01..ed860f69d7466 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/InMemoryFileIndex.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/InMemoryFileIndex.scala @@ -31,7 +31,6 @@ import org.apache.spark.internal.Logging import org.apache.spark.metrics.source.HiveCatalogMetrics import org.apache.spark.sql.SparkSession import org.apache.spark.sql.execution.streaming.FileStreamSink -import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types.StructType import org.apache.spark.util.SerializableConfiguration From 00ad219eafe4f1ec90e654b3d242316ba6783ddd Mon Sep 17 00:00:00 2001 From: gwang3 Date: Sun, 6 Oct 2019 21:10:30 +0800 Subject: [PATCH 9/9] Enhance comment --- .../scala/org/apache/spark/sql/internal/SQLConf.scala | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala index e4c54d8e2998b..34534dd14b3f2 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala @@ -834,9 +834,11 @@ object SQLConf { val IGNORE_DATA_LOCALITY = buildConf("spark.sql.sources.ignore.datalocality") - .doc("If it is set to true, Spark will not fetch the block locations for each file on " + - "listing files, which will boost listing file operation, at the cost of loss of " + - "data locality.") + .doc("If true, Spark will not fetch the block locations for each file on " + + "listing files. This speeds up file listing, but the scheduler cannot " + + "schedule tasks to take advantage of data locality. It can be particularly " + + "useful if data is read from a remote cluster so the scheduler could never " + + "take advantage of locality anyway.") .internal() .booleanConf .createWithDefault(false)