[SPARK-56870][SDP] Implement SCD1 Batch Processor; Extend Microbatch with CDC Metadata

common/utils/src/main/resources/error/error-conditions.json

@@ -209,6 +209,12 @@
  ],
  "sqlState" : "42703"
  },
+ "AUTOCDC_RESERVED_COLUMN_NAME_CONFLICT" : {
+ "message" : [
+ "Using <caseSensitivity> column name comparison, the column `<columnName>` in the <schemaName> schema conflicts with the reserved AutoCDC column name `<reservedColumnName>`. Rename or remove the column."
+ ],
+ "sqlState" : "42710"
+ },
  "AVRO_CANNOT_WRITE_NULL_FIELD" : {
  "message" : [
  "Cannot write null value for field <name> defined as non-null Avro data type <dataType>.",

sql/pipelines/src/main/scala/org/apache/spark/sql/pipelines/autocdc/Scd1BatchProcessor.scala

@@ -17,16 +17,26 @@
 
 package org.apache.spark.sql.pipelines.autocdc
 
-import org.apache.spark.sql.{functions => F}
+import org.apache.spark.SparkException
+import org.apache.spark.sql.{functions => F, AnalysisException}
+import org.apache.spark.sql.Column
 import org.apache.spark.sql.catalyst.util.QuotingUtils
 import org.apache.spark.sql.classic.DataFrame
+import org.apache.spark.sql.types.{DataType, StructField, StructType}
 import org.apache.spark.util.ArrayImplicits._
 
 /**
  * Per-microbatch processor for SCD Type 1 AutoCDC flows, complying to the specified [[changeArgs]]
  * configuration.
+ *
+ * @param changeArgs The CDC flow configuration.
+ * @param resolvedSequencingType The post-analysis [[DataType]] of the sequencing column, derived
+ * from the flow's resolved DataFrame at flow setup time.
  */
-case class Scd1BatchProcessor(changeArgs: ChangeArgs) {
+case class Scd1BatchProcessor(
+ changeArgs: ChangeArgs,
+ resolvedSequencingType: DataType) {
+
  /**
  * Deduplicate the incoming CDC microbatch by key, keeping the most recent event per key
  * as ordered by [[ChangeArgs.sequencing]].
@@ -59,9 +69,111 @@ case class Scd1BatchProcessor(changeArgs: ChangeArgs) {
  )
  .select(F.col(s"$winningRowCol.*"))
  }
+
+ /**
+ * Project the CDC metadata column onto the microbatch.
+ *
+ * This must run before any column selection is applied to the microbatch. The
+ * [[ChangeArgs.deleteCondition]] and [[ChangeArgs.sequencing]] expressions are evaluated against
+ * the current microbatch schema, and column selection may drop inputs required by those
+ * expressions.
+ *
+ * Rows are classified as deletes only when [[ChangeArgs.deleteCondition]] evaluates to true. A
+ * false or null delete condition classifies the row as an upsert.
+ *
+ * @param validatedMicrobatch A microbatch that has already been validated such that the
+ * sequencing column should not contain null values, and its data type
+ * should support ordering.
+ *
+ * The returned dataframe has all of the columns in the input microbatch + the CDC metadata
+ * column.
+ */
+ def extendMicrobatchRowsWithCdcMetadata(validatedMicrobatch: DataFrame): DataFrame = {
+ // Proactively validate the reserved CDC metadata column does not exist in the microbatch.
+ validateCdcMetadataColumnNotPresent(validatedMicrobatch)
+
+ val rowDeleteSequence: Column = changeArgs.deleteCondition match {
+ case Some(deleteCondition) =>
+ F.when(deleteCondition, changeArgs.sequencing).otherwise(F.lit(null))
+ case None =>
+ F.lit(null)
+ }
+
+ val rowUpsertSequence: Column =
+ // A row that is not a delete must be an upsert, these are mutually exclusive and a complete
+ // set of CDC event types.
+ F.when(rowDeleteSequence.isNull, changeArgs.sequencing).otherwise(F.lit(null))
+
+ validatedMicrobatch.withColumn(
+ Scd1BatchProcessor.cdcMetadataColName,
+ Scd1BatchProcessor.constructCdcMetadataCol(
+ deleteSequence = rowDeleteSequence,
+ upsertSequence = rowUpsertSequence,
+ sequencingType = resolvedSequencingType
+ )
+ )
+ }
+
+ private def validateCdcMetadataColumnNotPresent(microbatch: DataFrame): Unit = {
+ val microbatchSqlConf = microbatch.sparkSession.sessionState.conf
+ val resolver = microbatchSqlConf.resolver
+
+ microbatch.schema.fieldNames
+ .find(resolver(_, Scd1BatchProcessor.cdcMetadataColName))
+ .foreach { conflictingColumnName =>
+ throw new AnalysisException(
+ errorClass = "AUTOCDC_RESERVED_COLUMN_NAME_CONFLICT",
+ messageParameters = Map(
+ "caseSensitivity" -> CaseSensitivityLabels.of(microbatchSqlConf.caseSensitiveAnalysis),
+ "columnName" -> conflictingColumnName,
+ "schemaName" -> "microbatch",
+ "reservedColumnName" -> Scd1BatchProcessor.cdcMetadataColName
+ )
+ )
+ }
+ }
 }
 
 object Scd1BatchProcessor {
  // Columns prefixed with `__spark_autocdc_` are reserved for internal SDP AutoCDC processing.
- private[autocdc] val winningRowColName = "__spark_autocdc_winning_row"
+ private[autocdc] val winningRowColName: String = "__spark_autocdc_winning_row"
+ private[autocdc] val cdcMetadataColName: String = "__spark_autocdc_metadata"
+
+ private[autocdc] val cdcDeleteSequenceFieldName: String = "deleteSequence"
+ private[autocdc] val cdcUpsertSequenceFieldName: String = "upsertSequence"
+
+ /**
+ * Schema of the CDC metadata struct column for SCD1.
+ */
+ private def cdcMetadataColSchema(sequencingType: DataType): StructType =
+ StructType(
+ Seq(
+ // The sequencing of the event if it represents a delete, null otherwise.
+ StructField(cdcDeleteSequenceFieldName, sequencingType, nullable = true),
+ // The sequencing of the event if it represents an upsert, null otherwise.
+ StructField(cdcUpsertSequenceFieldName, sequencingType, nullable = true)
+ )
+ )
+
+ /**
+ * Construct the CDC metadata struct column for SCD1, following the exact schema and field
+ * ordering defined by [[cdcMetadataColSchema]].
+ */
+ private[autocdc] def constructCdcMetadataCol(
+ deleteSequence: Column,
+ upsertSequence: Column,
+ sequencingType: DataType): Column = {
+ val cdcMetadataFieldsInOrder = cdcMetadataColSchema(sequencingType).fields.map { field =>
+ val value = field.name match {
+ case `cdcDeleteSequenceFieldName` => deleteSequence
+ case `cdcUpsertSequenceFieldName` => upsertSequence
+ case other =>
+ throw SparkException.internalError(
+ s"Unable to construct SCD1 CDC metadata column due to unknown `${other}` field."
+ )
+ }
+ value.cast(field.dataType).as(field.name)
+ }
+ F.struct(cdcMetadataFieldsInOrder.toImmutableArraySeq: _*)
+ }
 }