From 0e1a8be90622822c7134e36f47ee9046595fcb1a Mon Sep 17 00:00:00 2001 From: lipeidian Date: Sat, 24 Jul 2021 12:32:02 +0800 Subject: [PATCH] Spark: Set properties for deletewriter --- .../org/apache/iceberg/spark/source/SparkAppenderFactory.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/spark/src/main/java/org/apache/iceberg/spark/source/SparkAppenderFactory.java b/spark/src/main/java/org/apache/iceberg/spark/source/SparkAppenderFactory.java index 29bb4edea174..96fcd8fc71a2 100644 --- a/spark/src/main/java/org/apache/iceberg/spark/source/SparkAppenderFactory.java +++ b/spark/src/main/java/org/apache/iceberg/spark/source/SparkAppenderFactory.java @@ -205,6 +205,7 @@ public EqualityDeleteWriter newEqDeleteWriter(EncryptedOutputFile f case PARQUET: return Parquet.writeDeletes(file.encryptingOutputFile()) .createWriterFunc(msgType -> SparkParquetWriters.buildWriter(lazyEqDeleteSparkType(), msgType)) + .setAll(properties) .overwrite() .rowSchema(eqDeleteRowSchema) .withSpec(spec) @@ -216,6 +217,7 @@ public EqualityDeleteWriter newEqDeleteWriter(EncryptedOutputFile f case AVRO: return Avro.writeDeletes(file.encryptingOutputFile()) .createWriterFunc(ignored -> new SparkAvroWriter(lazyEqDeleteSparkType())) + .setAll(properties) .overwrite() .rowSchema(eqDeleteRowSchema) .withSpec(spec) @@ -243,6 +245,7 @@ public PositionDeleteWriter newPosDeleteWriter(EncryptedOutputFile SparkSchemaUtil.convert(DeleteSchemaUtil.posDeleteSchema(posDeleteRowSchema)); return Parquet.writeDeletes(file.encryptingOutputFile()) .createWriterFunc(msgType -> SparkParquetWriters.buildWriter(sparkPosDeleteSchema, msgType)) + .setAll(properties) .overwrite() .rowSchema(posDeleteRowSchema) .withSpec(spec) @@ -254,6 +257,7 @@ public PositionDeleteWriter newPosDeleteWriter(EncryptedOutputFile case AVRO: return Avro.writeDeletes(file.encryptingOutputFile()) .createWriterFunc(ignored -> new SparkAvroWriter(lazyPosDeleteSparkType())) + .setAll(properties) .overwrite() .rowSchema(posDeleteRowSchema) .withSpec(spec)