From 0729d38985e3d618f2e9697c73c8001f17fa6b26 Mon Sep 17 00:00:00 2001 From: Liang-Chi Hsieh Date: Wed, 29 Jul 2026 08:04:46 -0700 Subject: [PATCH 1/2] [SPARK-58414][SQL][TESTS][FOLLOWUP] Use the imported LocalDateTime instead of the fully qualified name Addresses dongjoon-hyun's post-merge review: the suite already imports java.time.LocalDateTime, so drop the fully qualified references. Co-authored-by: Claude Code --- .../columnar/ArrowCachedBatchSerializerSuite.scala | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/columnar/ArrowCachedBatchSerializerSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/columnar/ArrowCachedBatchSerializerSuite.scala index 07672ad3e5cc5..9c793cd9329dd 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/columnar/ArrowCachedBatchSerializerSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/columnar/ArrowCachedBatchSerializerSuite.scala @@ -2283,15 +2283,15 @@ class ArrowCachedBatchSerializerSuite extends QueryTest with SharedSparkSession // the tagged-struct recognizers -- so the lossless struct representation must round-trip at // any nesting depth, including values outside the int64 epoch-nanos window (~1677-2262) // that the standard interchange encoding cannot represent. - val outOfWindow = java.time.LocalDateTime.of(3000, 1, 6, 12, 30, 45, 123456789) - val inWindow = java.time.LocalDateTime.of(2025, 1, 6, 12, 30, 45, 987654321) + val outOfWindow = LocalDateTime.of(3000, 1, 6, 12, 30, 45, 123456789) + val inWindow = LocalDateTime.of(2025, 1, 6, 12, 30, 45, 987654321) val nanosType = TimestampNTZNanosType(9) val arrayDf = singlePartDf( Seq(Seq(outOfWindow, inWindow)), ArrayType(nanosType)).cache() try { assert(arrayDf.count() == 1) - val read = arrayDf.collect().head.getSeq[java.time.LocalDateTime](0) + val read = arrayDf.collect().head.getSeq[LocalDateTime](0) assert(read == Seq(outOfWindow, inWindow), s"expected nested nanos to round-trip through an array, got: $read") } finally { @@ -2303,7 +2303,7 @@ class ArrowCachedBatchSerializerSuite extends QueryTest with SharedSparkSession Seq(Row(outOfWindow)), StructType(Seq(StructField("ts", nanosType)))).cache() try { assert(structDf.count() == 1) - val read = structDf.collect().head.getStruct(0).getAs[java.time.LocalDateTime](0) + val read = structDf.collect().head.getStruct(0).getAs[LocalDateTime](0) assert(read == outOfWindow, s"expected nested nanos to round-trip through a struct, got: $read") } finally { @@ -2315,7 +2315,7 @@ class ArrowCachedBatchSerializerSuite extends QueryTest with SharedSparkSession Seq(Map(1 -> outOfWindow)), MapType(IntegerType, nanosType)).cache() try { assert(mapDf.count() == 1) - val read = mapDf.collect().head.getMap[Int, java.time.LocalDateTime](0) + val read = mapDf.collect().head.getMap[Int, LocalDateTime](0) assert(read == Map(1 -> outOfWindow), s"expected nested nanos to round-trip through a map value, got: $read") } finally { From b1460c3814717cc4329dce7cadb178ed259b5f35 Mon Sep 17 00:00:00 2001 From: Liang-Chi Hsieh Date: Wed, 29 Jul 2026 08:12:55 -0700 Subject: [PATCH 2/2] [SPARK-58414][SQL][TESTS][FOLLOWUP] Run the nested nanos round trip under both vectorized-reader settings Addresses dongjoon-hyun's second post-merge review: wrap the test in the same Seq(false, true) CACHE_VECTORIZED_READER_ENABLED loop the neighboring round-trip tests use, since the suite's default pins the conf to false. Co-authored-by: Claude Code --- .../ArrowCachedBatchSerializerSuite.scala | 71 ++++++++++--------- 1 file changed, 38 insertions(+), 33 deletions(-) diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/columnar/ArrowCachedBatchSerializerSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/columnar/ArrowCachedBatchSerializerSuite.scala index 9c793cd9329dd..73c2b15daf2ff 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/columnar/ArrowCachedBatchSerializerSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/columnar/ArrowCachedBatchSerializerSuite.scala @@ -2282,45 +2282,50 @@ class ArrowCachedBatchSerializerSuite extends QueryTest with SharedSparkSession // accessor in ArrowColumnVector wraps its element vector through the constructor that runs // the tagged-struct recognizers -- so the lossless struct representation must round-trip at // any nesting depth, including values outside the int64 epoch-nanos window (~1677-2262) - // that the standard interchange encoding cannot represent. + // that the standard interchange encoding cannot represent. Like the neighboring round-trip + // tests, run under both vectorized-reader settings. val outOfWindow = LocalDateTime.of(3000, 1, 6, 12, 30, 45, 123456789) val inWindow = LocalDateTime.of(2025, 1, 6, 12, 30, 45, 987654321) val nanosType = TimestampNTZNanosType(9) - val arrayDf = singlePartDf( - Seq(Seq(outOfWindow, inWindow)), ArrayType(nanosType)).cache() - try { - assert(arrayDf.count() == 1) - val read = arrayDf.collect().head.getSeq[LocalDateTime](0) - assert(read == Seq(outOfWindow, inWindow), - s"expected nested nanos to round-trip through an array, got: $read") - } finally { - arrayDf.unpersist() - InMemoryRelation.clearSerializer() - } + Seq(false, true).foreach { vectorized => + withSQLConf(SQLConf.CACHE_VECTORIZED_READER_ENABLED.key -> vectorized.toString) { + val arrayDf = singlePartDf( + Seq(Seq(outOfWindow, inWindow)), ArrayType(nanosType)).cache() + try { + assert(arrayDf.count() == 1) + val read = arrayDf.collect().head.getSeq[LocalDateTime](0) + assert(read == Seq(outOfWindow, inWindow), + s"expected nested nanos to round-trip through an array, got: $read") + } finally { + arrayDf.unpersist() + InMemoryRelation.clearSerializer() + } - val structDf = singlePartDf( - Seq(Row(outOfWindow)), StructType(Seq(StructField("ts", nanosType)))).cache() - try { - assert(structDf.count() == 1) - val read = structDf.collect().head.getStruct(0).getAs[LocalDateTime](0) - assert(read == outOfWindow, - s"expected nested nanos to round-trip through a struct, got: $read") - } finally { - structDf.unpersist() - InMemoryRelation.clearSerializer() - } + val structDf = singlePartDf( + Seq(Row(outOfWindow)), StructType(Seq(StructField("ts", nanosType)))).cache() + try { + assert(structDf.count() == 1) + val read = structDf.collect().head.getStruct(0).getAs[LocalDateTime](0) + assert(read == outOfWindow, + s"expected nested nanos to round-trip through a struct, got: $read") + } finally { + structDf.unpersist() + InMemoryRelation.clearSerializer() + } - val mapDf = singlePartDf( - Seq(Map(1 -> outOfWindow)), MapType(IntegerType, nanosType)).cache() - try { - assert(mapDf.count() == 1) - val read = mapDf.collect().head.getMap[Int, LocalDateTime](0) - assert(read == Map(1 -> outOfWindow), - s"expected nested nanos to round-trip through a map value, got: $read") - } finally { - mapDf.unpersist() - InMemoryRelation.clearSerializer() + val mapDf = singlePartDf( + Seq(Map(1 -> outOfWindow)), MapType(IntegerType, nanosType)).cache() + try { + assert(mapDf.count() == 1) + val read = mapDf.collect().head.getMap[Int, LocalDateTime](0) + assert(read == Map(1 -> outOfWindow), + s"expected nested nanos to round-trip through a map value, got: $read") + } finally { + mapDf.unpersist() + InMemoryRelation.clearSerializer() + } + } } }