From da6aded7b50e430cc9fbf0c0c78750167f3ff8a6 Mon Sep 17 00:00:00 2001 From: Sun Dapeng Date: Sun, 19 Jul 2026 15:06:50 +0800 Subject: [PATCH 1/3] [core][spark] Support catalog-managed Format Table reads and writes Route managed Format Table partition discovery through the catalog so scans read exactly the registered partitions and an empty registration reads as empty instead of falling back to directory listing. Validate partition locations against the table path before any read or delete. Register written partitions at commit time in bounded batches, keep IF NOT EXISTS registration idempotent for concurrent writers, and fall back to unmanaged directory discovery when no catalog manages the table. --- docs/docs/flink/sql-ddl.md | 16 + docs/generated/core_configuration.html | 3 +- .../java/org/apache/paimon/CoreOptions.java | 5 +- .../apache/paimon/catalog/CachingCatalog.java | 30 +- .../org/apache/paimon/catalog/Catalog.java | 11 + .../apache/paimon/catalog/CatalogUtils.java | 102 +++- .../paimon/catalog/DelegateCatalog.java | 5 + .../org/apache/paimon/rest/RESTCatalog.java | 29 +- .../org/apache/paimon/table/FormatTable.java | 79 ++- .../table/format/FormatBatchWriteBuilder.java | 3 +- .../table/format/FormatReadBuilder.java | 10 + .../format/FormatTableCatalogProvider.java | 191 ++++++ .../table/format/FormatTableCommit.java | 148 ++++- .../paimon/table/format/FormatTableScan.java | 25 +- .../table/format/ManagedFormatTableScan.java | 209 +++++++ .../paimon/utils/PartitionPathUtils.java | 160 ++++- .../paimon/catalog/CachingCatalogTest.java | 26 + .../paimon/catalog/CatalogUtilsTest.java | 131 +++++ .../paimon/catalog/FileSystemCatalogTest.java | 52 ++ .../paimon/rest/MockRESTCatalogTest.java | 94 +++ .../apache/paimon/rest/RESTCatalogServer.java | 40 ++ .../apache/paimon/rest/RESTCatalogTest.java | 58 ++ .../table/FormatTableCompatibilityTest.java | 185 ++++++ .../table/format/FormatTableCommitTest.java | 247 ++++++++ .../format/ManagedFormatTableScanTest.java | 553 ++++++++++++++++++ .../paimon/utils/PartitionPathUtilsTest.java | 50 +- .../format/FormatTableBatchWriteBase.scala | 9 + .../format/FormatTableBatchWriteTest.scala | 98 ++++ 28 files changed, 2517 insertions(+), 52 deletions(-) create mode 100644 paimon-core/src/main/java/org/apache/paimon/table/format/FormatTableCatalogProvider.java create mode 100644 paimon-core/src/main/java/org/apache/paimon/table/format/ManagedFormatTableScan.java create mode 100644 paimon-core/src/test/java/org/apache/paimon/catalog/CatalogUtilsTest.java create mode 100644 paimon-core/src/test/java/org/apache/paimon/table/FormatTableCompatibilityTest.java create mode 100644 paimon-core/src/test/java/org/apache/paimon/table/format/FormatTableCommitTest.java create mode 100644 paimon-core/src/test/java/org/apache/paimon/table/format/ManagedFormatTableScanTest.java create mode 100644 paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/format/FormatTableBatchWriteTest.scala diff --git a/docs/docs/flink/sql-ddl.md b/docs/docs/flink/sql-ddl.md index 4964a0cc18d2..ff5dea32699e 100644 --- a/docs/docs/flink/sql-ddl.md +++ b/docs/docs/flink/sql-ddl.md @@ -34,6 +34,22 @@ Paimon catalogs currently support three types of metastores: See [CatalogOptions](../maintenance/configurations#catalogoptions) for detailed options when creating a catalog. +:::info + +For Format Tables, `metastore.partitioned-table = true` enables catalog-managed partitions, which +requires an internal table in a catalog that supports it (currently the REST catalog) and cannot +be combined with `format-table.implementation = engine`. The REST catalog validates this +combination on `CREATE TABLE` (catalog-level `table-default.*` options participate in the +effective options); other catalogs keep their previous behavior and treat the option as inert. + +A Format Table that carries `metastore.partitioned-table = true` where it cannot be honored +(e.g. in a Hive catalog, or on an external table) still loads — the option is ignored, the table +behaves as an unmanaged Format Table, and a warning is logged. To remove the option permanently, +use `ALTER TABLE my_table RESET ('metastore.partitioned-table')` where the catalog supports +altering the table. + +::: + ### Create Filesystem Catalog The following Flink SQL registers and uses a Paimon catalog named `my_catalog`. Metadata and table files are stored under `hdfs:///path/to/warehouse`. diff --git a/docs/generated/core_configuration.html b/docs/generated/core_configuration.html index 143c244451a9..f4e4098dd263 100644 --- a/docs/generated/core_configuration.html +++ b/docs/generated/core_configuration.html @@ -1029,7 +1029,8 @@ Boolean Whether to create this table as a partitioned table in metastore. For example, if you want to list all partitions of a Paimon table in Hive, you need to create this table as a partitioned table in Hive metastore. -This config option does not affect the default filesystem metastore. +This config option does not affect the default filesystem metastore. +For format tables in a REST catalog, setting this option makes partition metadata catalog-managed: partitions must be registered in the catalog, and partition listing does not fall back to filesystem discovery.
metastore.tag-to-partition
diff --git a/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java b/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java index 4f1cb3111193..96964c7fe72c 100644 --- a/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java +++ b/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java @@ -1823,7 +1823,10 @@ public String toString() { "Whether to create this table as a partitioned table in metastore.\n" + "For example, if you want to list all partitions of a Paimon table in Hive, " + "you need to create this table as a partitioned table in Hive metastore.\n" - + "This config option does not affect the default filesystem metastore."); + + "This config option does not affect the default filesystem metastore.\n" + + "For format tables in a REST catalog, setting this option makes partition " + + "metadata catalog-managed: partitions must be registered in the catalog, " + + "and partition listing does not fall back to filesystem discovery."); public static final ConfigOption METASTORE_TAG_TO_PARTITION = key("metastore.tag-to-partition") diff --git a/paimon-core/src/main/java/org/apache/paimon/catalog/CachingCatalog.java b/paimon-core/src/main/java/org/apache/paimon/catalog/CachingCatalog.java index 01f04cbba3ae..1b7ac137e036 100644 --- a/paimon-core/src/main/java/org/apache/paimon/catalog/CachingCatalog.java +++ b/paimon-core/src/main/java/org/apache/paimon/catalog/CachingCatalog.java @@ -338,9 +338,10 @@ public List listPartitions(Identifier identifier) throws TableNotExis @Override public void createPartitions(Identifier identifier, List> partitions) throws TableNotExistException { - wrapped.createPartitions(identifier, partitions); - if (partitionCache != null) { - partitionCache.invalidate(identifier); + try { + wrapped.createPartitions(identifier, partitions); + } finally { + invalidatePartitionCache(identifier); } } @@ -348,25 +349,34 @@ public void createPartitions(Identifier identifier, List> pa public void createPartitions( Identifier identifier, List> partitions, boolean ignoreIfExists) throws TableNotExistException { - wrapped.createPartitions(identifier, partitions, ignoreIfExists); - if (partitionCache != null) { - partitionCache.invalidate(identifier); + try { + wrapped.createPartitions(identifier, partitions, ignoreIfExists); + } finally { + invalidatePartitionCache(identifier); } } @Override public void dropPartitions(Identifier identifier, List> partitions) throws TableNotExistException { - wrapped.dropPartitions(identifier, partitions); - if (partitionCache != null) { - partitionCache.invalidate(identifier); + try { + wrapped.dropPartitions(identifier, partitions); + } finally { + invalidatePartitionCache(identifier); } } @Override public void alterPartitions(Identifier identifier, List partitions) throws TableNotExistException { - wrapped.alterPartitions(identifier, partitions); + try { + wrapped.alterPartitions(identifier, partitions); + } finally { + invalidatePartitionCache(identifier); + } + } + + private void invalidatePartitionCache(Identifier identifier) { if (partitionCache != null) { partitionCache.invalidate(identifier); } diff --git a/paimon-core/src/main/java/org/apache/paimon/catalog/Catalog.java b/paimon-core/src/main/java/org/apache/paimon/catalog/Catalog.java index 8e4ad8d51815..8693c8a259a5 100644 --- a/paimon-core/src/main/java/org/apache/paimon/catalog/Catalog.java +++ b/paimon-core/src/main/java/org/apache/paimon/catalog/Catalog.java @@ -670,6 +670,17 @@ default boolean supportsListTableByType() { return false; } + /** + * Whether this catalog can be the source of truth for partitions of managed Format Tables + * ({@code type=format-table} with {@code metastore.partitioned-table=true}), i.e. whether it + * implements the partition APIs that managed Format Table scans and commits rely on. Catalogs + * without this capability load such tables with the option ignored, falling back to filesystem + * partition discovery. + */ + default boolean supportsManagedFormatTablePartitions() { + return false; + } + // ==================== Version management methods ========================== /** diff --git a/paimon-core/src/main/java/org/apache/paimon/catalog/CatalogUtils.java b/paimon-core/src/main/java/org/apache/paimon/catalog/CatalogUtils.java index 0c34fc39b1d8..c8a4780d7f4c 100644 --- a/paimon-core/src/main/java/org/apache/paimon/catalog/CatalogUtils.java +++ b/paimon-core/src/main/java/org/apache/paimon/catalog/CatalogUtils.java @@ -39,6 +39,7 @@ import org.apache.paimon.table.FormatTable; import org.apache.paimon.table.Table; import org.apache.paimon.table.TableSnapshot; +import org.apache.paimon.table.format.FormatTableCatalogProvider; import org.apache.paimon.table.iceberg.IcebergTable; import org.apache.paimon.table.lance.LanceTable; import org.apache.paimon.table.object.ObjectTable; @@ -54,6 +55,9 @@ import org.apache.paimon.utils.Pair; import org.apache.paimon.utils.Preconditions; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import javax.annotation.Nullable; import java.util.ArrayList; @@ -85,6 +89,8 @@ /** Utils for {@link Catalog}. */ public class CatalogUtils { + private static final Logger LOG = LoggerFactory.getLogger(CatalogUtils.class); + public static Path path(String warehouse, String database, String table) { return new Path(String.format("%s/%s.db/%s", warehouse, database, table)); } @@ -173,6 +179,7 @@ private static void validateFormatTableOptions(Options options, boolean dataToke options.get(PRIMARY_KEY) == null, "Cannot define %s for format table.", PRIMARY_KEY.key()); + validateManagedFormatTableOptions(options); if (dataTokenEnabled && options.get(PATH) == null) { checkArgument( options.get(FORMAT_TABLE_IMPLEMENTATION) @@ -213,6 +220,44 @@ private static void validateFormatTableOptions(Options options, boolean dataToke } } + /** Validate options which are specific to catalog-managed format tables. */ + public static void validateManagedFormatTableOptions(Map tableOptions) { + validateManagedFormatTableOptions(Options.fromMap(tableOptions)); + } + + private static void validateManagedFormatTableOptions(Options options) { + if (options.get(CoreOptions.METASTORE_PARTITIONED_TABLE)) { + checkArgument( + options.get(FORMAT_TABLE_IMPLEMENTATION) + != CoreOptions.FormatTableImplementation.ENGINE, + "Cannot define %s=true together with %s=engine for a managed format table.", + CoreOptions.METASTORE_PARTITIONED_TABLE.key(), + FORMAT_TABLE_IMPLEMENTATION.key()); + } + } + + /** Validate that a catalog can persist the requested managed Format Table. */ + public static void validateManagedFormatTableCatalog( + Identifier identifier, + Map tableOptions, + Catalog catalog, + boolean isExternal) { + CoreOptions options = CoreOptions.fromMap(tableOptions); + if (options.type() != TableType.FORMAT_TABLE || !options.partitionedTableInMetastore()) { + return; + } + checkArgument( + !isExternal, + "Managed format table %s must be an internal table.", + identifier.getFullName()); + checkArgument( + catalog.supportsManagedFormatTablePartitions(), + "Managed format table %s requires a catalog with catalog-managed partition support" + + " (e.g. a REST catalog), but %s does not support it.", + identifier.getFullName(), + catalog.getClass().getSimpleName()); + } + public static void validateNamePattern(Catalog catalog, String namePattern) { if (Objects.nonNull(namePattern) && !catalog.supportsListByPattern()) { throw new UnsupportedOperationException( @@ -271,6 +316,34 @@ public static List listPartitionsFromFileSystem( return partitions; } + /** + * Why the managed Format Table semantics cannot be honored when loading the table from this + * catalog, or {@code null} if they can. + */ + @Nullable + private static String managedFormatTableLoadProblem( + Catalog catalog, TableMetadata metadata, Map tableOptions) { + if (!catalog.supportsManagedFormatTablePartitions()) { + return String.format( + "catalog %s does not support catalog-managed Format Table partitions", + catalog.getClass().getSimpleName()); + } + if (metadata.isExternal()) { + return "managed format tables must be internal tables"; + } + if (Options.fromMap(tableOptions).get(FORMAT_TABLE_IMPLEMENTATION) + == CoreOptions.FormatTableImplementation.ENGINE) { + return String.format( + "the option conflicts with %s=engine", FORMAT_TABLE_IMPLEMENTATION.key()); + } + if (catalog.catalogLoader() == null) { + return String.format( + "catalog %s does not provide a catalog loader", + catalog.getClass().getSimpleName()); + } + return null; + } + /** * Load table from {@link Catalog}, this table can be: * @@ -304,7 +377,30 @@ public static Table loadTable( Function dataFileIO = metadata.isExternal() ? externalFileIO : internalFileIO; if (options.type() == TableType.FORMAT_TABLE) { - return toFormatTable(identifier, schema, dataFileIO, catalogContext); + FormatTableCatalogProvider catalogProvider = null; + if (options.partitionedTableInMetastore()) { + String problem = managedFormatTableLoadProblem(catalog, metadata, schema.options()); + if (problem == null) { + catalogProvider = + new FormatTableCatalogProvider(identifier, catalog.catalogLoader()); + } else { + // Existing tables may carry the option from a version (or catalog) where it + // was inert; refusing to load them would leave no SQL way to repair the + // table. Keep them readable with filesystem partition discovery instead. + LOG.warn( + "Ignoring '{}=true' on format table {}: {}. Falling back to " + + "filesystem partition discovery, the table behaves as an " + + "unmanaged format table. Use ALTER TABLE ... RESET/UNSET " + + "to remove the option.", + CoreOptions.METASTORE_PARTITIONED_TABLE.key(), + identifier, + problem); + Map effectiveOptions = new HashMap<>(schema.options()); + effectiveOptions.remove(CoreOptions.METASTORE_PARTITIONED_TABLE.key()); + schema = schema.copy(effectiveOptions); + } + } + return toFormatTable(identifier, schema, dataFileIO, catalogContext, catalogProvider); } if (options.type() == TableType.OBJECT_TABLE) { @@ -453,7 +549,8 @@ private static FormatTable toFormatTable( Identifier identifier, TableSchema schema, Function fileIO, - CatalogContext catalogContext) { + CatalogContext catalogContext, + @Nullable FormatTableCatalogProvider catalogProvider) { Map options = schema.options(); FormatTable.Format format = FormatTable.parseFormat( @@ -471,6 +568,7 @@ private static FormatTable toFormatTable( .options(options) .comment(schema.comment()) .catalogContext(catalogContext) + .catalogProvider(catalogProvider) .build(); } diff --git a/paimon-core/src/main/java/org/apache/paimon/catalog/DelegateCatalog.java b/paimon-core/src/main/java/org/apache/paimon/catalog/DelegateCatalog.java index 290849024fdb..8685fde9d7fc 100644 --- a/paimon-core/src/main/java/org/apache/paimon/catalog/DelegateCatalog.java +++ b/paimon-core/src/main/java/org/apache/paimon/catalog/DelegateCatalog.java @@ -198,6 +198,11 @@ public boolean supportsListTableByType() { return wrapped.supportsListTableByType(); } + @Override + public boolean supportsManagedFormatTablePartitions() { + return wrapped.supportsManagedFormatTablePartitions(); + } + @Override public boolean supportsVersionManagement() { return wrapped.supportsVersionManagement(); diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java b/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java index c44378f6d00f..9f0737f868fe 100644 --- a/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java +++ b/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java @@ -63,6 +63,7 @@ import org.apache.paimon.table.Instant; import org.apache.paimon.table.Table; import org.apache.paimon.table.TableSnapshot; +import org.apache.paimon.table.format.FormatTableCatalogProvider; import org.apache.paimon.table.sink.BatchTableCommit; import org.apache.paimon.table.system.SystemTableLoader; import org.apache.paimon.utils.Pair; @@ -95,6 +96,8 @@ import static org.apache.paimon.catalog.CatalogUtils.isSystemDatabase; import static org.apache.paimon.catalog.CatalogUtils.listPartitionsFromFileSystem; import static org.apache.paimon.catalog.CatalogUtils.validateCreateTable; +import static org.apache.paimon.catalog.CatalogUtils.validateManagedFormatTableCatalog; +import static org.apache.paimon.catalog.CatalogUtils.validateManagedFormatTableOptions; import static org.apache.paimon.options.CatalogOptions.CASE_SENSITIVE; /** A catalog implementation for REST. */ @@ -453,6 +456,11 @@ public boolean supportsListTableByType() { return true; } + @Override + public boolean supportsManagedFormatTablePartitions() { + return true; + } + @Override public boolean supportsVersionManagement() { return true; @@ -567,8 +575,13 @@ public void createTable(Identifier identifier, Schema schema, boolean ignoreIfEx checkNotBranch(identifier, "createTable"); checkNotSystemTable(identifier, "createTable"); validateCreateTable(schema, dataTokenEnabled); - createExternalTablePathIfNotExist(schema); tableDefaultOptions.forEach(schema.options()::putIfAbsent); + // Defaults participate in the managed Format Table combination, so validate the + // effective options rather than only the explicit ones. + validateManagedFormatTableOptions(schema.options()); + validateManagedFormatTableCatalog( + identifier, schema.options(), this, schema.options().containsKey(PATH.key())); + createExternalTablePathIfNotExist(schema); Schema newSchema = inferSchemaIfExternalPaimonTable(schema); api.createTable(identifier, newSchema); } catch (AlreadyExistsException e) { @@ -645,8 +658,14 @@ public void replaceTable(Identifier identifier, Schema newSchema, boolean ignore checkNotBranch(identifier, "replaceTable"); checkNotSystemTable(identifier, "replaceTable"); validateCreateTable(newSchema, dataTokenEnabled); + tableDefaultOptions.forEach(newSchema.options()::putIfAbsent); + // Defaults participate in the managed Format Table combination, so validate the + // effective options rather than only the explicit ones. Externality is not decided + // client-side here: a round-tripped schema of an internal table may carry the synthetic + // path option, and the server remains the authority for replace semantics. + validateManagedFormatTableOptions(newSchema.options()); + validateManagedFormatTableCatalog(identifier, newSchema.options(), this, false); try { - tableDefaultOptions.forEach(newSchema.options()::putIfAbsent); api.replaceTable(identifier, newSchema); } catch (NoSuchResourceException e) { if (!ignoreIfNotExists) { @@ -751,6 +770,9 @@ public void createPartitions( identifier, e.getMessage())); } catch (BadRequestException e) { throw new IllegalArgumentException(e.getMessage()); + } finally { + // The server may have committed even when the response was lost or malformed. + FormatTableCatalogProvider.advanceGeneration(identifier); } } @@ -769,6 +791,9 @@ public void dropPartitions(Identifier identifier, List> part throw new TableNoPermissionException(identifier, e); } catch (BadRequestException e) { throw new IllegalArgumentException(e.getMessage()); + } finally { + // The server may have committed even when the response was lost or malformed. + FormatTableCatalogProvider.advanceGeneration(identifier); } return; } diff --git a/paimon-core/src/main/java/org/apache/paimon/table/FormatTable.java b/paimon-core/src/main/java/org/apache/paimon/table/FormatTable.java index 9a832176caaf..64325e70b7c8 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/FormatTable.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/FormatTable.java @@ -18,9 +18,11 @@ package org.apache.paimon.table; +import org.apache.paimon.CoreOptions; import org.apache.paimon.Snapshot; import org.apache.paimon.annotation.Public; import org.apache.paimon.catalog.CatalogContext; +import org.apache.paimon.catalog.CatalogUtils; import org.apache.paimon.catalog.Identifier; import org.apache.paimon.fs.FileIO; import org.apache.paimon.manifest.IndexManifestEntry; @@ -29,6 +31,7 @@ import org.apache.paimon.stats.Statistics; import org.apache.paimon.table.format.FormatBatchWriteBuilder; import org.apache.paimon.table.format.FormatReadBuilder; +import org.apache.paimon.table.format.FormatTableCatalogProvider; import org.apache.paimon.table.sink.BatchWriteBuilder; import org.apache.paimon.table.sink.StreamWriteBuilder; import org.apache.paimon.table.source.BatchVectorSearchBuilder; @@ -56,8 +59,9 @@ * operations on this table allow for reading or writing to these files, facilitating the retrieval * of existing data and the addition of new files. * - *

Partitioned file format table just like the standard hive format. Partitions are discovered - * and inferred based on directory structure. + *

A partitioned file format table uses the standard Hive directory layout. By default, + * partitions are discovered from that layout. Catalog-managed format tables use catalog metadata + * for partition visibility while retaining the same physical layout. * * @since 0.9.0 */ @@ -75,6 +79,12 @@ public interface FormatTable extends Table { CatalogContext catalogContext(); + /** Catalog access used by managed partition discovery and registration. */ + @Nullable + default FormatTableCatalogProvider catalogProvider() { + return null; + } + /** Currently supported formats. */ enum Format { ORC, @@ -115,6 +125,7 @@ class Builder { private Map options; @Nullable private String comment; private CatalogContext catalogContext; + @Nullable private FormatTableCatalogProvider catalogProvider; public Builder fileIO(FileIO fileIO) { this.fileIO = fileIO; @@ -161,6 +172,11 @@ public Builder catalogContext(CatalogContext catalogContext) { return this; } + public Builder catalogProvider(@Nullable FormatTableCatalogProvider catalogProvider) { + this.catalogProvider = catalogProvider; + return this; + } + public FormatTable build() { return new FormatTableImpl( fileIO, @@ -171,7 +187,8 @@ public FormatTable build() { format, options, comment, - catalogContext); + catalogContext, + catalogProvider); } } @@ -189,6 +206,7 @@ class FormatTableImpl implements FormatTable { private final Map options; @Nullable private final String comment; private CatalogContext catalogContext; + @Nullable private final FormatTableCatalogProvider catalogProvider; public FormatTableImpl( FileIO fileIO, @@ -200,6 +218,30 @@ public FormatTableImpl( Map options, @Nullable String comment, CatalogContext catalogContext) { + this( + fileIO, + identifier, + rowType, + partitionKeys, + location, + format, + options, + comment, + catalogContext, + null); + } + + public FormatTableImpl( + FileIO fileIO, + Identifier identifier, + RowType rowType, + List partitionKeys, + String location, + Format format, + Map options, + @Nullable String comment, + CatalogContext catalogContext, + @Nullable FormatTableCatalogProvider catalogProvider) { this.fileIO = fileIO; this.identifier = identifier; this.rowType = rowType; @@ -209,6 +251,7 @@ public FormatTableImpl( this.options = options; this.comment = comment; this.catalogContext = catalogContext; + this.catalogProvider = catalogProvider; } @Override @@ -265,6 +308,27 @@ public FileIO fileIO() { public FormatTable copy(Map dynamicOptions) { Map newOptions = new HashMap<>(options); newOptions.putAll(dynamicOptions); + + CoreOptions coreOptions = CoreOptions.fromMap(options); + CoreOptions copiedCoreOptions = CoreOptions.fromMap(newOptions); + boolean managed = coreOptions.partitionedTableInMetastore(); + boolean copiedManaged = copiedCoreOptions.partitionedTableInMetastore(); + if (managed != copiedManaged) { + throw new IllegalArgumentException( + String.format( + "Dynamic option '%s' cannot change whether Format Table partitions are catalog-managed.", + CoreOptions.METASTORE_PARTITIONED_TABLE.key())); + } + if (managed + && coreOptions.formatTablePartitionOnlyValueInPath() + != copiedCoreOptions.formatTablePartitionOnlyValueInPath()) { + throw new IllegalArgumentException( + String.format( + "Dynamic option '%s' cannot change the physical partition layout of a catalog-managed Format Table.", + CoreOptions.FORMAT_TABLE_PARTITION_ONLY_VALUE_IN_PATH.key())); + } + CatalogUtils.validateManagedFormatTableOptions(newOptions); + return new FormatTableImpl( fileIO, identifier, @@ -274,7 +338,8 @@ public FormatTable copy(Map dynamicOptions) { format, newOptions, comment, - catalogContext); + catalogContext, + catalogProvider); } @Override @@ -302,6 +367,12 @@ public FullTextSearchBuilder newFullTextSearchBuilder() { public CatalogContext catalogContext() { return this.catalogContext; } + + @Override + @Nullable + public FormatTableCatalogProvider catalogProvider() { + return catalogProvider; + } } @Override diff --git a/paimon-core/src/main/java/org/apache/paimon/table/format/FormatBatchWriteBuilder.java b/paimon-core/src/main/java/org/apache/paimon/table/format/FormatBatchWriteBuilder.java index 924e67dbdfdc..d3970bb878ea 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/format/FormatBatchWriteBuilder.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/format/FormatBatchWriteBuilder.java @@ -87,7 +87,8 @@ public BatchTableCommit newCommit() { Identifier.fromString(table.fullName()), staticPartition, syncHiveUri, - table.catalogContext()); + table.catalogContext(), + table.catalogProvider()); } @Override diff --git a/paimon-core/src/main/java/org/apache/paimon/table/format/FormatReadBuilder.java b/paimon-core/src/main/java/org/apache/paimon/table/format/FormatReadBuilder.java index 30a567b436a6..1c9010d0be04 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/format/FormatReadBuilder.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/format/FormatReadBuilder.java @@ -173,6 +173,16 @@ public TableScan newScan() { partitionFilter = partitionPredicateOpt.get(); } } + if (options.partitionedTableInMetastore()) { + if (table.catalogProvider() == null) { + throw new IllegalStateException( + String.format( + "Managed format table %s has no catalog partition provider. " + + "The catalog client does not support managed format tables.", + table.fullName())); + } + return new ManagedFormatTableScan(table, partitionFilter, limit); + } return new FormatTableScan(table, partitionFilter, limit); } diff --git a/paimon-core/src/main/java/org/apache/paimon/table/format/FormatTableCatalogProvider.java b/paimon-core/src/main/java/org/apache/paimon/table/format/FormatTableCatalogProvider.java new file mode 100644 index 000000000000..c297ef63e094 --- /dev/null +++ b/paimon-core/src/main/java/org/apache/paimon/table/format/FormatTableCatalogProvider.java @@ -0,0 +1,191 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.table.format; + +import org.apache.paimon.PagedList; +import org.apache.paimon.annotation.Experimental; +import org.apache.paimon.catalog.Catalog; +import org.apache.paimon.catalog.CatalogLoader; +import org.apache.paimon.catalog.Identifier; +import org.apache.paimon.partition.Partition; +import org.apache.paimon.utils.StringUtils; + +import org.apache.paimon.shade.caffeine2.com.github.benmanes.caffeine.cache.Cache; +import org.apache.paimon.shade.caffeine2.com.github.benmanes.caffeine.cache.Caffeine; + +import javax.annotation.Nullable; + +import java.io.Serializable; +import java.time.Duration; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.atomic.AtomicLong; + +/** Serializable catalog access for a managed format table. */ +@Experimental +public class FormatTableCatalogProvider implements Serializable { + + private static final long serialVersionUID = 1L; + + private static final int PARTITION_PAGE_SIZE = 1000; + private static final int MAX_CACHED_PATTERNS = 128; + private static final int MAX_TRACKED_TABLE_GENERATIONS = 10_000; + private static final Duration PARTITION_CACHE_TTL = Duration.ofSeconds(30); + private static final Duration TABLE_GENERATION_TTL = Duration.ofHours(1); + + // Keyed by identifier full name: listings themselves live in per-instance caches, so sharing + // a generation counter across table incarnations (or across catalogs using the same name) can + // only cause an extra invalidation, never a stale read. + private static final Cache GENERATIONS = + Caffeine.newBuilder() + .expireAfterAccess(TABLE_GENERATION_TTL) + .maximumSize(MAX_TRACKED_TABLE_GENERATIONS) + .executor(Runnable::run) + .build(); + + private final Identifier identifier; + private final CatalogLoader catalogLoader; + + @Nullable private transient Cache> partitionCache; + // Reused across list/create calls: constructing a Catalog (HTTP client, auth provider, and + // for some auth providers an initial token fetch) per partition operation is expensive, and + // RESTCatalog.close() is a no-op so a long-lived instance leaks nothing. Recreated lazily + // after deserialization. + @Nullable private transient Catalog catalog; + + public FormatTableCatalogProvider(Identifier identifier, CatalogLoader catalogLoader) { + this.identifier = identifier; + this.catalogLoader = catalogLoader; + } + + /** List every catalog-visible partition matching the optional name prefix pattern. */ + public List listPartitions(@Nullable String partitionNamePattern) { + return cache().get( + partitionCacheKey(partitionNamePattern), + ignored -> loadPartitions(partitionNamePattern)); + } + + /** + * Advance the partition-listing generation of every provider of this table in this JVM, so + * same-process scans skip cached listings taken before the mutation. The catalog partition DDL + * path (create/dropPartitions) calls this from a {@code finally} block: the mutation's outcome + * can be ambiguous (the server may have committed even when the response was lost), so the + * cached listing must be dropped after every attempt, not only after a confirmed success. + */ + public static void advanceGeneration(Identifier identifier) { + GENERATIONS.get(identifier.getFullName(), ignored -> new AtomicLong()).incrementAndGet(); + } + + /** Create partitions with the catalog's idempotent create contract. */ + public void createPartitions(List> partitions) { + if (partitions.isEmpty()) { + return; + } + try { + // Register in bounded batches: one backfill commit can touch tens of thousands of + // partitions (e.g. years of hourly data in a single dynamic-partition INSERT), and a + // single unbounded request either monopolizes a catalog worker or trips server-side + // request caps, failing the job after its data files are already committed. + // Registration is an idempotent ADD-only upsert, so a mid-way failure leaves a state + // that a rerun or MSCK converges from. + for (int start = 0; start < partitions.size(); start += PARTITION_PAGE_SIZE) { + catalog() + .createPartitions( + identifier, + partitions.subList( + start, + Math.min(start + PARTITION_PAGE_SIZE, partitions.size()))); + } + } catch (RuntimeException e) { + throw e; + } catch (Exception e) { + throw new RuntimeException( + String.format( + "Failed to register partitions for managed format table %s.", + identifier), + e); + } finally { + advanceGeneration(identifier); + cache().invalidateAll(); + } + } + + private List loadPartitions(@Nullable String partitionNamePattern) { + List partitions = new ArrayList<>(); + Set seenPageTokens = new HashSet<>(); + try { + Catalog catalog = catalog(); + String pageToken = null; + do { + PagedList page = + catalog.listPartitionsPaged( + identifier, PARTITION_PAGE_SIZE, pageToken, partitionNamePattern); + partitions.addAll(page.getElements()); + pageToken = page.getNextPageToken(); + if (StringUtils.isNotEmpty(pageToken) && !seenPageTokens.add(pageToken)) { + throw new IllegalStateException( + String.format( + "Catalog returned repeated partition page token '%s' for managed format table %s.", + pageToken, identifier.getFullName())); + } + } while (StringUtils.isNotEmpty(pageToken)); + return Collections.unmodifiableList(partitions); + } catch (RuntimeException e) { + throw e; + } catch (Exception e) { + throw new RuntimeException( + String.format( + "Failed to list partitions for managed format table %s.", identifier), + e); + } + } + + private synchronized Catalog catalog() { + if (catalog == null) { + catalog = catalogLoader.load(); + } + return catalog; + } + + private synchronized Cache> cache() { + if (partitionCache == null) { + partitionCache = + Caffeine.newBuilder() + .expireAfterWrite(PARTITION_CACHE_TTL) + .maximumSize(MAX_CACHED_PATTERNS) + .executor(Runnable::run) + .build(); + } + return partitionCache; + } + + private String partitionCacheKey(@Nullable String partitionNamePattern) { + // Partition mutations advance a process-local generation so providers in this JVM skip + // stale entries. Providers in other JVMs converge when PARTITION_CACHE_TTL expires. + return generation().get() + "\000" + partitionNamePattern; + } + + private AtomicLong generation() { + return GENERATIONS.get(identifier.getFullName(), ignored -> new AtomicLong()); + } +} diff --git a/paimon-core/src/main/java/org/apache/paimon/table/format/FormatTableCommit.java b/paimon-core/src/main/java/org/apache/paimon/table/format/FormatTableCommit.java index ed98c6ba50ff..8d38388907c5 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/format/FormatTableCommit.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/format/FormatTableCommit.java @@ -48,7 +48,6 @@ import java.util.List; import java.util.Map; import java.util.Set; -import java.util.StringJoiner; import static org.apache.paimon.table.format.FormatBatchWriteBuilder.validateStaticPartition; @@ -63,6 +62,10 @@ public class FormatTableCommit implements BatchTableCommit { protected boolean overwrite = false; private Catalog hiveCatalog; private Identifier tableIdentifier; + @Nullable private final FormatTableCatalogProvider catalogProvider; + // Volatile: commit() and abort() may run on different threads when callers drive the API + // directly; abort must observe that registration already began to keep its no-op guarantee. + private volatile boolean partitionRegistrationStarted; public FormatTableCommit( String location, @@ -74,6 +77,30 @@ public FormatTableCommit( @Nullable Map staticPartitions, @Nullable String syncHiveUri, CatalogContext catalogContext) { + this( + location, + partitionKeys, + fileIO, + formatTablePartitionOnlyValueInPath, + overwrite, + tableIdentifier, + staticPartitions, + syncHiveUri, + catalogContext, + null); + } + + public FormatTableCommit( + String location, + List partitionKeys, + FileIO fileIO, + boolean formatTablePartitionOnlyValueInPath, + boolean overwrite, + Identifier tableIdentifier, + @Nullable Map staticPartitions, + @Nullable String syncHiveUri, + CatalogContext catalogContext, + @Nullable FormatTableCatalogProvider catalogProvider) { this.location = location; this.fileIO = fileIO; this.formatTablePartitionOnlyValueInPath = formatTablePartitionOnlyValueInPath; @@ -82,6 +109,7 @@ public FormatTableCommit( this.overwrite = overwrite; this.partitionKeys = partitionKeys; this.tableIdentifier = tableIdentifier; + this.catalogProvider = catalogProvider; if (syncHiveUri != null) { try { Options options = new Options(); @@ -143,7 +171,9 @@ public void commit(List commitMessages) { for (TwoPhaseOutputStream.Committer committer : committers) { committer.commit(this.fileIO); - if (partitionKeys != null && !partitionKeys.isEmpty() && hiveCatalog != null) { + if (partitionKeys != null + && !partitionKeys.isEmpty() + && (hiveCatalog != null || catalogProvider != null)) { partitionSpecs.add( extractPartitionSpecFromPath( committer.targetPath().getParent(), partitionKeys)); @@ -152,6 +182,14 @@ public void commit(List commitMessages) { for (TwoPhaseOutputStream.Committer committer : committers) { committer.clean(this.fileIO); } + if (catalogProvider != null && !partitionSpecs.isEmpty()) { + partitionRegistrationStarted = true; + try { + catalogProvider.createPartitions(new ArrayList<>(partitionSpecs)); + } catch (RuntimeException e) { + throw partitionRegistrationFailure(e); + } + } for (Map partitionSpec : partitionSpecs) { if (hiveCatalog != null) { try { @@ -172,11 +210,28 @@ public void commit(List commitMessages) { } } catch (Exception e) { - this.abort(commitMessages); - throw new RuntimeException(e); + if (!partitionRegistrationStarted) { + this.abort(commitMessages); + throw new RuntimeException(e); + } + throw e instanceof RuntimeException ? (RuntimeException) e : new RuntimeException(e); } } + private RuntimeException partitionRegistrationFailure(RuntimeException cause) { + String tableName = tableIdentifier.getFullName(); + return new RuntimeException( + String.format( + "Managed partition registration failed for %s after data files were " + + "committed. Committed data files were preserved because the " + + "catalog state may be ambiguous. Verify the catalog partition " + + "metadata and, if partitions are missing, re-run a partition " + + "metadata sync for %s (e.g. the Spark procedure " + + "sys.sync_format_table_metadata or MSCK REPAIR TABLE).", + tableName, tableName), + cause); + } + private Method getHiveCreatePartitionsInHmsMethod() throws NoSuchMethodException { Method hiveCreatePartitionsInHmsMethod = hiveCatalog @@ -192,12 +247,68 @@ private Method getHiveCreatePartitionsInHmsMethod() throws NoSuchMethodException private LinkedHashMap extractPartitionSpecFromPath( Path partitionPath, List partitionKeys) { - if (formatTablePartitionOnlyValueInPath) { - return PartitionPathUtils.extractPartitionSpecFromPathOnlyValue( - partitionPath, partitionKeys); - } else { - return PartitionPathUtils.extractPartitionSpecFromPath(partitionPath); + // The writer always lays partitions out as //.../, so the partition + // spec is exactly the trailing partitionKeys.size() components of the partition + // directory. Never walk beyond them: the table location itself may contain foreign + // 'k=v' segments that must not leak into the registered spec. Registered values are + // RAW (unescaped); readers re-escape them when probing partition directories. + String[] components = new String[partitionKeys.size()]; + Path current = partitionPath; + for (int i = partitionKeys.size() - 1; i >= 0; i--) { + if (current == null || current.getName().isEmpty()) { + throw new IllegalArgumentException( + String.format( + "Partition path '%s' has fewer than %s directory levels required " + + "by partition keys %s of table %s.", + partitionPath, + partitionKeys.size(), + partitionKeys, + tableIdentifier.getFullName())); + } + components[i] = current.getName(); + current = current.getParent(); + } + + LinkedHashMap partitionSpec = new LinkedHashMap<>(); + for (int i = 0; i < partitionKeys.size(); i++) { + String expectedKey = partitionKeys.get(i); + String component = components[i]; + if (formatTablePartitionOnlyValueInPath) { + partitionSpec.put(expectedKey, PartitionPathUtils.unescapePathName(component)); + } else { + int splitIndex = component.indexOf('='); + if (splitIndex < 0) { + throw new IllegalArgumentException( + String.format( + "Partition directory '%s' of partition path '%s' is not in " + + "'key=value' form expected for partition key '%s' " + + "of table %s.", + component, + partitionPath, + expectedKey, + tableIdentifier.getFullName())); + } + String parsedKey = + PartitionPathUtils.unescapePathName(component.substring(0, splitIndex)); + String parsedValue = + PartitionPathUtils.unescapePathName(component.substring(splitIndex + 1)); + if (!expectedKey.equals(parsedKey)) { + throw new IllegalArgumentException( + String.format( + "Partition directory '%s' of partition path '%s' declares " + + "partition key '%s' but partition key '%s' was " + + "expected at position %s for table %s.", + component, + partitionPath, + parsedKey, + expectedKey, + i, + tableIdentifier.getFullName())); + } + partitionSpec.put(expectedKey, parsedValue); + } } + return partitionSpec; } private static Path buildPartitionPath( @@ -208,24 +319,29 @@ private static Path buildPartitionPath( if (partitionSpec.isEmpty() || partitionKeys.isEmpty()) { throw new IllegalArgumentException("partitionSpec or partitionKeys is empty."); } - StringJoiner joiner = new StringJoiner("/"); + LinkedHashMap orderedSpec = new LinkedHashMap<>(); for (int i = 0; i < partitionSpec.size(); i++) { String key = partitionKeys.get(i); if (partitionSpec.containsKey(key)) { - if (formatTablePartitionOnlyValueInPath) { - joiner.add(partitionSpec.get(key)); - } else { - joiner.add(key + "=" + partitionSpec.get(key)); - } + orderedSpec.put(key, partitionSpec.get(key)); } else { throw new RuntimeException("partitionSpec does not contain key: " + key); } } - return new Path(location, joiner.toString()); + return new Path( + location, + PartitionPathUtils.generatePartitionPathUtil( + orderedSpec, formatTablePartitionOnlyValueInPath)); } @Override public void abort(List commitMessages) { + // Once partition registration has started the data files are already committed and must be + // preserved: the catalog outcome is ambiguous (see partitionRegistrationFailure), so + // discarding the files here could delete data the catalog already references. + if (partitionRegistrationStarted) { + return; + } try { for (CommitMessage commitMessage : commitMessages) { if (commitMessage instanceof TwoPhaseCommitMessage) { diff --git a/paimon-core/src/main/java/org/apache/paimon/table/format/FormatTableScan.java b/paimon-core/src/main/java/org/apache/paimon/table/format/FormatTableScan.java index cb0edefc82fc..62fc6b09e3c5 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/format/FormatTableScan.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/format/FormatTableScan.java @@ -59,6 +59,7 @@ import javax.annotation.Nullable; +import java.io.FileNotFoundException; import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; @@ -81,9 +82,9 @@ public class FormatTableScan implements InnerTableScan { private static final Logger LOG = LoggerFactory.getLogger(FormatTableScan.class); - private final FormatTable table; - private final CoreOptions coreOptions; - @Nullable private PartitionPredicate partitionFilter; + protected final FormatTable table; + protected final CoreOptions coreOptions; + @Nullable protected PartitionPredicate partitionFilter; @Nullable private final Integer limit; private final long targetSplitSize; private final long openFileCost; @@ -142,7 +143,7 @@ public static boolean isDataFileName(String fileName) { return fileName != null && !fileName.startsWith(".") && !fileName.startsWith("_"); } - private BinaryRow toPartitionRow(LinkedHashMap partitionSpec) { + protected BinaryRow toPartitionRow(LinkedHashMap partitionSpec) { RowType partitionType = table.partitionType(); GenericRow row = convertSpecToInternalRow(partitionSpec, partitionType, table.defaultPartName()); @@ -160,7 +161,11 @@ public List splits() { LinkedHashMap partitionSpec = pair.getKey(); BinaryRow partitionRow = toPartitionRow(partitionSpec); if (partitionFilter == null || partitionFilter.test(partitionRow)) { - splits.addAll(createSplits(fileIO, pair.getValue(), partitionRow)); + try { + splits.addAll(createSplits(fileIO, pair.getValue(), partitionRow)); + } catch (FileNotFoundException e) { + onPartitionFileNotFound(partitionSpec, pair.getValue(), e); + } } } } else { @@ -177,7 +182,7 @@ public List splits() { } } - List, Path>> findPartitions() { + protected List, Path>> findPartitions() { LOG.debug( "Find partitions for format table {}, partition filter: {}", table.name(), @@ -222,6 +227,14 @@ List, Path>> findPartitions() { } } + protected void onPartitionFileNotFound( + LinkedHashMap partitionSpec, + Path partitionPath, + FileNotFoundException exception) + throws IOException { + throw exception; + } + protected static List, Path>> generatePartitions( List partitionKeys, RowType partitionType, diff --git a/paimon-core/src/main/java/org/apache/paimon/table/format/ManagedFormatTableScan.java b/paimon-core/src/main/java/org/apache/paimon/table/format/ManagedFormatTableScan.java new file mode 100644 index 000000000000..7e16dee0839e --- /dev/null +++ b/paimon-core/src/main/java/org/apache/paimon/table/format/ManagedFormatTableScan.java @@ -0,0 +1,209 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.table.format; + +import org.apache.paimon.fs.FileStatus; +import org.apache.paimon.fs.Path; +import org.apache.paimon.manifest.PartitionEntry; +import org.apache.paimon.partition.Partition; +import org.apache.paimon.partition.PartitionPredicate; +import org.apache.paimon.predicate.Predicate; +import org.apache.paimon.table.FormatTable; +import org.apache.paimon.utils.Pair; +import org.apache.paimon.utils.PartitionPathUtils; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.annotation.Nullable; + +import java.io.FileNotFoundException; +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; + +/** A format table scan whose partition visibility is owned by a catalog. */ +public class ManagedFormatTableScan extends FormatTableScan { + + private static final Logger LOG = LoggerFactory.getLogger(ManagedFormatTableScan.class); + + private final FormatTableCatalogProvider catalogProvider; + + public ManagedFormatTableScan( + FormatTable table, + @Nullable PartitionPredicate partitionFilter, + @Nullable Integer limit) { + super(table, partitionFilter, limit); + FormatTableCatalogProvider provider = table.catalogProvider(); + if (provider == null) { + throw new IllegalStateException( + String.format( + "Managed format table %s has no catalog partition provider.", + table.fullName())); + } + this.catalogProvider = provider; + } + + @Override + protected List, Path>> findPartitions() { + String partitionNamePattern = createPartitionNamePattern(); + List partitions = catalogProvider.listPartitions(partitionNamePattern); + if (partitions.isEmpty() && partitionNamePattern == null) { + warnIfFilesystemPartitionsExist(); + } + List, Path>> result = new ArrayList<>(partitions.size()); + Path tablePath = new Path(table.location()); + // Do not trust the catalog to be duplicate-free: a repeated spec would double every split + // of that partition and silently duplicate query results. + Set seenPartitionPaths = new HashSet<>(partitions.size()); + for (Partition partition : partitions) { + LinkedHashMap partitionSpec = normalizeSpec(partition.spec()); + String partitionPath = + PartitionPathUtils.generatePartitionPathUtil( + partitionSpec, coreOptions.formatTablePartitionOnlyValueInPath()); + if (!seenPartitionPaths.add(partitionPath)) { + continue; + } + result.add(Pair.of(partitionSpec, new Path(tablePath, partitionPath))); + } + return result; + } + + @Override + public List listPartitionEntries() { + List partitions = catalogProvider.listPartitions(null); + if (partitions.isEmpty()) { + warnIfFilesystemPartitionsExist(); + } + List entries = new ArrayList<>(partitions.size()); + for (Partition partition : partitions) { + entries.add( + new PartitionEntry( + toPartitionRow(normalizeSpec(partition.spec())), + partition.recordCount(), + partition.fileSizeInBytes(), + partition.fileCount(), + partition.lastFileCreationTime(), + partition.totalBuckets())); + } + return entries; + } + + /** + * Warn when the catalog knows no partitions but the table directory contains subdirectories: + * typically a table that predates managed partition support (or was written by a client that + * does not register partitions) and needs a metadata sync before its data becomes visible. + */ + private void warnIfFilesystemPartitionsExist() { + try { + for (FileStatus status : table.fileIO().listStatus(new Path(table.location()))) { + if (status.isDir() && !status.getPath().getName().startsWith(".")) { + LOG.warn( + "Managed format table {} has no partitions registered in the catalog " + + "but its location {} contains directories. Data written " + + "before enabling catalog-managed partitions (or by clients " + + "that do not register partitions) is invisible until the " + + "partition metadata is synced, e.g. with the Spark procedure " + + "sys.sync_format_table_metadata or MSCK REPAIR TABLE.", + table.fullName(), + table.location()); + return; + } + } + } catch (IOException ignored) { + // Best-effort hint only; never fail or slow down the scan because of it. + } + } + + /** + * Build the partition-name prefix pattern pushed down to the catalog, or {@code null} to list + * all partitions without pushdown. See {@link + * PartitionPathUtils#buildPartitionNamePrefixPattern} for the pattern contract; this returns + * {@code null} when there is no partition predicate, when the predicate has no leading equality + * prefix, or when the prefix cannot be expressed in the pattern contract. + */ + @Nullable + String createPartitionNamePattern() { + Optional predicate = extractPartitionPredicate(partitionFilter); + if (!predicate.isPresent()) { + return null; + } + + Map equalityPrefix = + extractLeadingEqualityPartitionSpecWhenOnlyAnd( + table.partitionKeys(), predicate.get(), table.partitionType()); + return PartitionPathUtils.buildPartitionNamePrefixPattern( + table.partitionKeys(), equalityPrefix); + } + + private LinkedHashMap normalizeSpec(Map spec) { + if (spec == null + || spec.size() != table.partitionKeys().size() + || !spec.keySet().containsAll(table.partitionKeys())) { + throw corruptPartitionSpec(spec); + } + LinkedHashMap normalized = new LinkedHashMap<>(); + boolean onlyValueInPath = coreOptions.formatTablePartitionOnlyValueInPath(); + for (String partitionKey : table.partitionKeys()) { + String value = spec.get(partitionKey); + // Catalog metadata is not trusted for path construction. In a value-only layout, + // reject complete path components such as '.' and '..' that would escape the table. + try { + PartitionPathUtils.validatePartitionValueForPath(value, onlyValueInPath); + } catch (IllegalArgumentException e) { + throw corruptPartitionSpec(spec); + } + normalized.put(partitionKey, value); + } + return normalized; + } + + private IllegalStateException corruptPartitionSpec(@Nullable Map spec) { + return new IllegalStateException( + String.format( + "Catalog returned corrupt partition metadata %s for managed format table %s; " + + "expected exactly the partition keys %s with values usable as " + + "path components.", + spec, table.fullName(), table.partitionKeys())); + } + + @Override + protected void onPartitionFileNotFound( + LinkedHashMap partitionSpec, + Path partitionPath, + FileNotFoundException exception) { + // A registered partition without a directory reads as empty, matching Hive semantics + // (e.g. ADD PARTITION before the first INSERT). Warn so genuine drift — a directory + // removed behind the catalog's back — is still discoverable. + LOG.warn( + "Partition '{}' of managed format table {} is registered in the catalog but its " + + "directory '{}' does not exist; treating the partition as empty. If the " + + "directory was removed on purpose, drop the partition or repair the " + + "metadata, e.g. with the Spark procedure sys.sync_format_table_metadata " + + "or MSCK REPAIR TABLE.", + PartitionPathUtils.generatePartitionName(partitionSpec, false), + table.fullName(), + partitionPath); + } +} diff --git a/paimon-core/src/main/java/org/apache/paimon/utils/PartitionPathUtils.java b/paimon-core/src/main/java/org/apache/paimon/utils/PartitionPathUtils.java index fb7baf5b06b2..b17168cb3af2 100644 --- a/paimon-core/src/main/java/org/apache/paimon/utils/PartitionPathUtils.java +++ b/paimon-core/src/main/java/org/apache/paimon/utils/PartitionPathUtils.java @@ -109,13 +109,105 @@ public static String generatePartitionPathUtil( suffixBuf.append(escapePathName(e.getKey())); suffixBuf.append('='); } - suffixBuf.append(escapePathName(e.getValue())); + String value = e.getValue(); + validatePartitionValueForPath(value, onlyValue); + suffixBuf.append(escapePathName(value)); i++; } suffixBuf.append(Path.SEPARATOR); return suffixBuf.toString(); } + /** + * Generate a partition path without the trailing separator, e.g. {@code dt=20250101/hr=01}. + * This is the canonical partition name used when talking to a partition-managing catalog. + */ + public static String generatePartitionName( + LinkedHashMap partitionSpec, boolean onlyValue) { + String path = generatePartitionPathUtil(partitionSpec, onlyValue); + return path.endsWith(Path.SEPARATOR) + ? path.substring(0, path.length() - Path.SEPARATOR.length()) + : path; + } + + /** + * Validate that a partition value is safe for the configured path layout. In a key-value + * layout, values such as {@code "."} are part of a component such as {@code "pt=."} and are + * safe. In a value-only layout, {@code "."} and {@code ".."} are complete path components and + * would resolve to a different directory. + */ + public static void validatePartitionValueForPath(String value, boolean onlyValueInPath) { + if (value == null + || value.isEmpty() + || (onlyValueInPath && (".".equals(value) || "..".equals(value)))) { + throw new IllegalArgumentException( + String.format( + "Partition value '%s' cannot be used as a partition path component.", + value)); + } + } + + /** Conservatively validate a value when the physical partition layout is unknown. */ + public static void validatePartitionValueForPath(String value) { + validatePartitionValueForPath(value, true); + } + + /** Validate every value of a partition spec for the configured path layout. */ + public static void validatePartitionSpecForPath( + Map partitionSpec, boolean onlyValueInPath) { + for (String value : partitionSpec.values()) { + validatePartitionValueForPath(value, onlyValueInPath); + } + } + + /** Conservatively validate a spec when the physical partition layout is unknown. */ + public static void validatePartitionSpecForPath(Map partitionSpec) { + validatePartitionSpecForPath(partitionSpec, true); + } + + /** + * Build the partition-name prefix pattern pushed down to a partition-managing catalog from the + * leading equality prefix of a partition predicate. + * + *

Pattern contract (shared by every engine talking to the catalog): partition names are the + * escaped {@code key=value} form joined by {@code '/'}; {@code '%'} is the only wildcard and + * there is no escape sequence for it ({@code '_'} stays a literal). A complete spec matches the + * exact partition name; an incomplete prefix is suffixed with {@code '%'}. + * + *

Returns {@code null} whenever pushdown must be skipped and the caller should list all + * partitions instead: the equality prefix is empty, a prefix value is blank, or the escaped + * prefix contains a literal {@code '%'} that the contract cannot express. + */ + @Nullable + public static String buildPartitionNamePrefixPattern( + List partitionKeys, Map equalityPrefix) { + if (equalityPrefix.isEmpty()) { + return null; + } + LinkedHashMap orderedPrefix = new LinkedHashMap<>(); + for (String partitionKey : partitionKeys) { + if (!equalityPrefix.containsKey(partitionKey)) { + break; + } + String value = equalityPrefix.get(partitionKey); + if (StringUtils.isNullOrWhitespaceOnly(value)) { + return null; + } + orderedPrefix.put(partitionKey, value); + } + if (orderedPrefix.isEmpty()) { + return null; + } + String escapedPrefix = generatePartitionPath(orderedPrefix); + if (escapedPrefix.indexOf('%') >= 0) { + return null; + } + if (orderedPrefix.size() == partitionKeys.size()) { + return escapedPrefix.substring(0, escapedPrefix.length() - 1); + } + return escapedPrefix + '%'; + } + public static List generatePartitionPaths( List> partitions, RowType partitionType) { return partitions.stream() @@ -265,12 +357,45 @@ public static LinkedHashMap extractPartitionSpecFromPath(Path cu return fullPartSpec; } + /** Extract exactly the trailing key-value components for the declared partition keys. */ + @Nullable + static LinkedHashMap extractPartitionSpecFromPath( + Path currPath, List partitionKeys) { + String[] values = new String[partitionKeys.size()]; + Path current = currPath; + for (int i = partitionKeys.size() - 1; i >= 0; i--) { + if (current == null) { + return null; + } + Matcher matcher = PARTITION_NAME_PATTERN.matcher(current.getName()); + if (!matcher.matches() + || !partitionKeys.get(i).equals(unescapePathName(matcher.group(1)))) { + return null; + } + values[i] = unescapePathName(matcher.group(2)); + current = current.getParent(); + } + + LinkedHashMap spec = new LinkedHashMap<>(); + for (int i = 0; i < partitionKeys.size(); i++) { + spec.put(partitionKeys.get(i), values[i]); + } + return spec; + } + public static LinkedHashMap extractPartitionSpecFromPathOnlyValue( Path currPath, List partitionKeys) { LinkedHashMap fullPartSpec = new LinkedHashMap<>(); String[] split = currPath.toString().split(Path.SEPARATOR); for (int i = 0; i < partitionKeys.size(); i++) { - fullPartSpec.put(partitionKeys.get(i), split[split.length - partitionKeys.size() + i]); + // Unescape the directory component so the extracted value is the RAW partition value, + // consistent with the key=value branch (extractPartitionSpecFromPath) and with the + // values the write path registers into a partition-managing catalog. Without this, + // directories containing escaped characters (e.g. a%3Ab) would round-trip to a + // different value than the one registered (a:b). + fullPartSpec.put( + partitionKeys.get(i), + unescapePathName(split[split.length - partitionKeys.size() + i])); } return fullPartSpec; } @@ -330,8 +455,9 @@ public static List, Path>> searchPartSpecAndP part.getPath(), partitionKeys), part.getPath())); } else { - LinkedHashMap spec = extractPartitionSpecFromPath(part.getPath()); - if (spec.size() != partitionKeys.size()) { + LinkedHashMap spec = + extractPartitionSpecFromPath(part.getPath(), partitionKeys); + if (spec == null) { // illegal path, for example: /path/to/table/tmp/unknown, path without "=" continue; } @@ -360,8 +486,17 @@ private static FileStatus[] getFileStatusRecurse( GenericRow values = partitionType == null ? null : new GenericRow(partitionType.getFieldCount()); + FileStatus fileStatus; + try { + fileStatus = fileIO.getFileStatus(path); + } catch (FileNotFoundException e) { + // A missing root simply means the table has no partitions yet. + return new FileStatus[0]; + } catch (IOException e) { + throw new RuntimeException("Failed to list files in " + path, e); + } + try { - FileStatus fileStatus = fileIO.getFileStatus(path); // Skip partition levels already fixed by the scan-path prefix. int levelOffset = partitionKeys.size() - expectLevel; listStatusRecursively( @@ -378,9 +513,10 @@ private static FileStatus[] getFileStatusRecurse( defaultPartValue, levelOffset, values); - } catch (FileNotFoundException e) { - return new FileStatus[0]; } catch (IOException e) { + // Never degrade a mid-scan failure into an empty listing: callers diff this result + // against partition metadata and an incomplete listing would deregister partitions + // that still exist. throw new RuntimeException("Failed to list files in " + path, e); } @@ -412,7 +548,15 @@ private static void listStatusRecursively( } if (fileStatus.isDir()) { - for (FileStatus stat : fileIO.listStatus(fileStatus.getPath())) { + FileStatus[] children; + try { + children = fileIO.listStatus(fileStatus.getPath()); + } catch (FileNotFoundException e) { + // The directory vanished after the parent listed it: the partitions beneath it + // are gone, skipping just this subtree keeps the rest of the listing complete. + return; + } + for (FileStatus stat : children) { int partitionKeyIndex = levelOffset + level; String partitionKey = partitionKeys.get(partitionKeyIndex); diff --git a/paimon-core/src/test/java/org/apache/paimon/catalog/CachingCatalogTest.java b/paimon-core/src/test/java/org/apache/paimon/catalog/CachingCatalogTest.java index 0961050f110d..3e91251ae812 100644 --- a/paimon-core/src/test/java/org/apache/paimon/catalog/CachingCatalogTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/catalog/CachingCatalogTest.java @@ -368,6 +368,32 @@ public void testCreatePartitionsWithIgnoreIfExistsInvalidatesPartitionCache() th assertThat(catalog.listPartitions(identifier)).containsExactly(created); } + @Test + public void testCreatePartitionsFailureInvalidatesPartitionCache() throws Exception { + Catalog wrapped = Mockito.mock(Catalog.class); + TestableCachingCatalog catalog = + new TestableCachingCatalog(wrapped, EXPIRATION_TTL, ticker); + Identifier identifier = new Identifier("db", "tbl"); + Map spec = singletonMap("dt", "20260717"); + Partition created = new Partition(spec, 0, 0, 0, 0, -1, false); + List stored = new ArrayList<>(); + when(wrapped.listPartitions(identifier)).thenAnswer(ignored -> new ArrayList<>(stored)); + RuntimeException responseLost = new RuntimeException("response lost"); + Mockito.doAnswer( + ignored -> { + stored.add(created); + throw responseLost; + }) + .when(wrapped) + .createPartitions(identifier, singletonList(spec)); + + assertThat(catalog.listPartitions(identifier)).isEmpty(); + assertThatThrownBy(() -> catalog.createPartitions(identifier, singletonList(spec))) + .isSameAs(responseLost); + + assertThat(catalog.listPartitions(identifier)).containsExactly(created); + } + @Test public void testDeadlock() throws Exception { Catalog underlyCatalog = this.catalog; diff --git a/paimon-core/src/test/java/org/apache/paimon/catalog/CatalogUtilsTest.java b/paimon-core/src/test/java/org/apache/paimon/catalog/CatalogUtilsTest.java new file mode 100644 index 000000000000..edcc79af157c --- /dev/null +++ b/paimon-core/src/test/java/org/apache/paimon/catalog/CatalogUtilsTest.java @@ -0,0 +1,131 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.catalog; + +import org.apache.paimon.fs.local.LocalFileIO; +import org.apache.paimon.schema.Schema; +import org.apache.paimon.schema.TableSchema; +import org.apache.paimon.table.FormatTable; +import org.apache.paimon.table.Table; +import org.apache.paimon.types.DataTypes; + +import org.junit.jupiter.api.Test; + +import static org.apache.paimon.CoreOptions.FILE_FORMAT; +import static org.apache.paimon.CoreOptions.FORMAT_TABLE_IMPLEMENTATION; +import static org.apache.paimon.CoreOptions.METASTORE_PARTITIONED_TABLE; +import static org.apache.paimon.CoreOptions.PATH; +import static org.apache.paimon.CoreOptions.TYPE; +import static org.apache.paimon.TableType.FORMAT_TABLE; +import static org.apache.paimon.catalog.CatalogUtils.loadTable; +import static org.apache.paimon.catalog.CatalogUtils.validateCreateTable; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** Tests for {@link CatalogUtils}. */ +class CatalogUtilsTest { + + @Test + void testRejectEngineImplementationForManagedFormatTableOnCreate() { + Schema schema = + Schema.newBuilder() + .column("id", DataTypes.INT()) + .option(TYPE.key(), FORMAT_TABLE.toString()) + .option(METASTORE_PARTITIONED_TABLE.key(), "true") + .option(FORMAT_TABLE_IMPLEMENTATION.key(), "engine") + .build(); + + assertThatThrownBy(() -> validateCreateTable(schema, false)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining(METASTORE_PARTITIONED_TABLE.key()) + .hasMessageContaining(FORMAT_TABLE_IMPLEMENTATION.key()); + } + + @Test + void testEngineImplementationManagedFormatTableLoadsAsUnmanaged() throws Exception { + // The option combination is invalid for managed partitions, but an existing table must + // stay loadable: it degrades to an unmanaged format table instead of failing. + Table table = loadManagedFormatTable("engine", true); + assertUnmanagedFormatTable(table); + } + + @Test + void testManagedFormatTableOutsideCapableCatalogLoadsAsUnmanaged() throws Exception { + // A catalog that cannot manage partitions (e.g. Hive) must not reject existing tables that + // carry the (previously inert) option; it loads them as unmanaged format tables. + Table table = loadManagedFormatTable("paimon", false); + assertUnmanagedFormatTable(table); + } + + @Test + void testLoadManagedFormatTableFromCapableCatalog() throws Exception { + Table table = loadManagedFormatTable("paimon", true); + assertThat(table).isInstanceOf(FormatTable.class); + FormatTable formatTable = (FormatTable) table; + assertThat( + new org.apache.paimon.CoreOptions(formatTable.options()) + .partitionedTableInMetastore()) + .isTrue(); + assertThat(formatTable.catalogProvider()).isNotNull(); + } + + private static void assertUnmanagedFormatTable(Table table) { + assertThat(table).isInstanceOf(FormatTable.class); + FormatTable formatTable = (FormatTable) table; + assertThat( + new org.apache.paimon.CoreOptions(formatTable.options()) + .partitionedTableInMetastore()) + .isFalse(); + assertThat(formatTable.catalogProvider()).isNull(); + } + + private static Table loadManagedFormatTable( + String formatTableImplementation, boolean supportsManagedPartitions) throws Exception { + Identifier identifier = Identifier.create("managed_db", "managed_table"); + Schema schema = + Schema.newBuilder() + .column("id", DataTypes.INT()) + .column("dt", DataTypes.STRING()) + .partitionKeys("dt") + .option(TYPE.key(), FORMAT_TABLE.toString()) + .option(METASTORE_PARTITIONED_TABLE.key(), "true") + .option(FORMAT_TABLE_IMPLEMENTATION.key(), formatTableImplementation) + .option(FILE_FORMAT.key(), "parquet") + .option(PATH.key(), "file:///tmp/managed_db.db/managed_table") + .build(); + TableMetadata metadata = + new TableMetadata(TableSchema.create(0, schema), false, "managed-table-id"); + Catalog catalog = mock(Catalog.class); + when(catalog.catalogLoader()).thenReturn(() -> catalog); + when(catalog.supportsManagedFormatTablePartitions()).thenReturn(supportsManagedPartitions); + + return loadTable( + catalog, + identifier, + path -> new LocalFileIO(), + path -> new LocalFileIO(), + ignored -> metadata, + null, + null, + null, + supportsManagedPartitions); + } +} diff --git a/paimon-core/src/test/java/org/apache/paimon/catalog/FileSystemCatalogTest.java b/paimon-core/src/test/java/org/apache/paimon/catalog/FileSystemCatalogTest.java index 04f67c018d1b..7bb122e1fea6 100644 --- a/paimon-core/src/test/java/org/apache/paimon/catalog/FileSystemCatalogTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/catalog/FileSystemCatalogTest.java @@ -18,9 +18,15 @@ package org.apache.paimon.catalog; +import org.apache.paimon.CoreOptions; +import org.apache.paimon.TableType; import org.apache.paimon.fs.Path; import org.apache.paimon.options.Options; import org.apache.paimon.schema.Schema; +import org.apache.paimon.schema.SchemaChange; +import org.apache.paimon.schema.SchemaManager; +import org.apache.paimon.table.FormatTable; +import org.apache.paimon.table.Table; import org.apache.paimon.types.DataTypes; import org.apache.paimon.shade.guava30.com.google.common.collect.Lists; @@ -28,6 +34,7 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; /** Tests for {@link FileSystemCatalog}. */ @@ -77,4 +84,49 @@ public void testAlterDatabase() throws Exception { false)) .isInstanceOf(UnsupportedOperationException.class); } + + @Test + public void testResetManagedPartitionsOnStrandedFormatTable() throws Exception { + String database = "stranded_managed_format_table_db"; + Identifier identifier = Identifier.create(database, "stranded_format_table"); + catalog.createDatabase(database, false); + // Write the schema file directly to simulate a format table stranded with the + // REST-only option in a filesystem catalog. + Schema schema = + Schema.newBuilder() + .column("id", DataTypes.INT()) + .column("dt", DataTypes.STRING()) + .partitionKeys("dt") + .option(CoreOptions.TYPE.key(), TableType.FORMAT_TABLE.toString()) + .option(CoreOptions.FILE_FORMAT.key(), "parquet") + .option(CoreOptions.METASTORE_PARTITIONED_TABLE.key(), "true") + .build(); + Path tablePath = + ((FileSystemCatalog) DelegateCatalog.rootCatalog(catalog)) + .getTableLocation(identifier); + new SchemaManager(fileIO, tablePath).createTable(schema); + // A catalog that cannot manage partitions must still load such a stranded table: the + // option is ignored and the table degrades to an unmanaged format table (readable), rather + // than becoming permanently unreadable. + Table stranded = catalog.getTable(identifier); + assertThat(stranded).isInstanceOf(FormatTable.class); + assertThat(new CoreOptions(stranded.options()).partitionedTableInMetastore()).isFalse(); + + // Alters that do not touch managed-sensitive options skip the managed validation. + catalog.alterTable( + identifier, + Lists.newArrayList(SchemaChange.setOption("custom.unrelated-option", "value")), + false); + + // The documented remediation must succeed on a non-REST catalog. + catalog.alterTable( + identifier, + Lists.newArrayList( + SchemaChange.removeOption(CoreOptions.METASTORE_PARTITIONED_TABLE.key())), + false); + + assertThat(catalog.getTable(identifier).options()) + .doesNotContainKey(CoreOptions.METASTORE_PARTITIONED_TABLE.key()) + .containsEntry("custom.unrelated-option", "value"); + } } diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/MockRESTCatalogTest.java b/paimon-core/src/test/java/org/apache/paimon/rest/MockRESTCatalogTest.java index b4af3194b72f..c05f45875ed4 100644 --- a/paimon-core/src/test/java/org/apache/paimon/rest/MockRESTCatalogTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/rest/MockRESTCatalogTest.java @@ -26,6 +26,8 @@ import org.apache.paimon.catalog.CatalogContext; import org.apache.paimon.catalog.Identifier; import org.apache.paimon.catalog.TableQueryAuthResult; +import org.apache.paimon.fs.Path; +import org.apache.paimon.fs.local.LocalFileIO; import org.apache.paimon.options.CatalogOptions; import org.apache.paimon.options.Options; import org.apache.paimon.predicate.FieldRef; @@ -46,6 +48,8 @@ import org.apache.paimon.rest.exceptions.NotImplementedException; import org.apache.paimon.rest.responses.ConfigResponse; import org.apache.paimon.schema.Schema; +import org.apache.paimon.table.FormatTable; +import org.apache.paimon.table.format.FormatTableCatalogProvider; import org.apache.paimon.types.DataTypes; import org.apache.paimon.types.RowType; import org.apache.paimon.utils.JsonSerdeUtil; @@ -247,6 +251,96 @@ void testManagedFormatTablePartitionListingDoesNotFallback() throws Exception { .isInstanceOf(NotImplementedException.class); } + @Test + void testCreatePartitionFailureInvalidatesManagedListingCache() throws Exception { + Identifier identifier = createManagedFormatTable(); + FormatTable table = (FormatTable) restCatalog.getTable(identifier); + FormatTableCatalogProvider provider = table.catalogProvider(); + assertThat(provider).isNotNull(); + assertThat(provider.listPartitions(null)).isEmpty(); + Map partition = Collections.singletonMap("dt", "20260717"); + restCatalogServer.failNextCreatePartitionsResponse(); + + assertThatThrownBy( + () -> + restCatalog.createPartitions( + identifier, Collections.singletonList(partition))) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("Injected response failure"); + + assertThat(provider.listPartitions(null)) + .extracting(org.apache.paimon.partition.Partition::spec) + .containsExactly(partition); + } + + @Test + void testDropPartitionFailureInvalidatesManagedListingCache() throws Exception { + Identifier identifier = createManagedFormatTable(); + FormatTable table = (FormatTable) restCatalog.getTable(identifier); + FormatTableCatalogProvider provider = table.catalogProvider(); + assertThat(provider).isNotNull(); + Map partition = Collections.singletonMap("dt", "20260717"); + restCatalog.createPartitions(identifier, Collections.singletonList(partition)); + assertThat(provider.listPartitions(null)) + .extracting(org.apache.paimon.partition.Partition::spec) + .containsExactly(partition); + restCatalogServer.failNextDropPartitionsResponse(); + + assertThatThrownBy( + () -> + restCatalog.dropPartitions( + identifier, Collections.singletonList(partition))) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("Injected response failure"); + + assertThat(provider.listPartitions(null)).isEmpty(); + } + + @Test + void testRejectExternalManagedFormatTableBeforeCreate() throws Exception { + Identifier identifier = Identifier.create("db1", "external_managed_format_table"); + restCatalog.createDatabase(identifier.getDatabaseName(), true); + String externalPath = dataPath + "/external-managed-format-table"; + Schema schema = + Schema.newBuilder() + .option(CoreOptions.TYPE.key(), TableType.FORMAT_TABLE.toString()) + .option(CoreOptions.METASTORE_PARTITIONED_TABLE.key(), "true") + .option(CoreOptions.FILE_FORMAT.key(), "parquet") + .option(CoreOptions.PATH.key(), externalPath) + .column("id", DataTypes.INT()) + .column("dt", DataTypes.STRING()) + .partitionKeys("dt") + .build(); + + assertThatThrownBy(() -> restCatalog.createTable(identifier, schema, false)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("internal table"); + assertThat(restCatalog.listTables(identifier.getDatabaseName())) + .doesNotContain(identifier.getTableName()); + assertThat(LocalFileIO.create().exists(new Path(externalPath))).isFalse(); + } + + @Test + void testRoundTrippedManagedFormatTableReplacePassesClientValidation() throws Exception { + Identifier identifier = createManagedFormatTable(); + FormatTable existing = (FormatTable) restCatalog.getTable(identifier); + Schema replacement = + Schema.newBuilder() + .column("id", DataTypes.INT()) + .column("dt", DataTypes.STRING()) + .partitionKeys("dt") + .options(existing.options()) + .build(); + assertThat(replacement.options()) + .containsEntry(CoreOptions.PATH.key(), existing.location()); + + // The mock service does not implement Format Table replacement. Reaching that response + // proves the REST client accepted the unchanged synthetic path from the loaded table. + assertThatThrownBy(() -> restCatalog.replaceTable(identifier, replacement, false)) + .isInstanceOf(UnsupportedOperationException.class) + .hasMessageContaining("replaceTable does not support format tables"); + } + private Identifier createManagedFormatTable() throws Exception { Identifier identifier = Identifier.create("db1", "managed_partition_table"); restCatalog.createDatabase(identifier.getDatabaseName(), true); diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogServer.java b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogServer.java index 3a0e1539eea1..385b27cb9fbf 100644 --- a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogServer.java +++ b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogServer.java @@ -212,6 +212,9 @@ public class RESTCatalogServer { private final List> receivedHeaders = new ArrayList<>(); private volatile boolean partitionListingSupported = true; + private volatile boolean failNextCreatePartitionsResponse; + private volatile boolean failNextDropPartitionsResponse; + private volatile int tableGetCount; public RESTCatalogServer( String dataPath, AuthProvider authProvider, ConfigResponse config, String warehouse) { @@ -279,6 +282,22 @@ public void setPartitionListingSupported(boolean partitionListingSupported) { this.partitionListingSupported = partitionListingSupported; } + public void failNextCreatePartitionsResponse() { + this.failNextCreatePartitionsResponse = true; + } + + public void failNextDropPartitionsResponse() { + this.failNextDropPartitionsResponse = true; + } + + public int tableGetCount() { + return tableGetCount; + } + + public void resetTableGetCount() { + tableGetCount = 0; + } + public void addNoPermissionDatabase(String database) { noPermissionDatabases.add(database); } @@ -1733,6 +1752,7 @@ private MockResponse tableHandle(String method, String data, Identifier identifi } switch (method) { case "GET": + tableGetCount++; TableMetadata tableMetadata; if (identifier.isSystemTable()) { TableSchema schema = catalog.loadTableSchema(identifier); @@ -1897,6 +1917,16 @@ private MockResponse partitionsApiHandle( existed.add(spec); } } + if (failNextCreatePartitionsResponse) { + failNextCreatePartitionsResponse = false; + return mockResponse( + new ErrorResponse( + ErrorResponse.RESOURCE_TYPE_PARTITION, + tableIdentifier.getFullName(), + "Injected response failure after creating partitions.", + 400), + 400); + } return mockResponse(new CreatePartitionsResponse(created, existed), 200); default: return new MockResponse().setResponseCode(404); @@ -1940,6 +1970,16 @@ private MockResponse dropPartitionsHandle(String data, Identifier tableIdentifie } return false; }); + if (failNextDropPartitionsResponse) { + failNextDropPartitionsResponse = false; + return mockResponse( + new ErrorResponse( + ErrorResponse.RESOURCE_TYPE_PARTITION, + tableIdentifier.getFullName(), + "Injected response failure after dropping partitions.", + 400), + 400); + } return mockResponse(new DropPartitionsResponse(dropped, missing), 200); } diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTest.java b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTest.java index 504dc6fab6ec..3f13c1cab844 100644 --- a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTest.java @@ -76,6 +76,7 @@ import org.apache.paimon.schema.SchemaChange; import org.apache.paimon.schema.SchemaManager; import org.apache.paimon.table.FileStoreTable; +import org.apache.paimon.table.FormatTable; import org.apache.paimon.table.Instant; import org.apache.paimon.table.Table; import org.apache.paimon.table.TableSnapshot; @@ -1639,6 +1640,63 @@ void testDropPartitionsLoadsNonManagedTableOnce() throws Exception { Mockito.verify(catalogSpy, Mockito.times(1)).getTable(identifier); } + @Test + void testManagedFormatTableCommitAndScanMatchesFileSystemMode() throws Exception { + Identifier identifier = Identifier.create("format_partition_db", "managed_scan_table"); + catalog.createDatabase(identifier.getDatabaseName(), true); + catalog.createTable( + identifier, + Schema.newBuilder() + .option(CoreOptions.TYPE.key(), TableType.FORMAT_TABLE.toString()) + .option(METASTORE_PARTITIONED_TABLE.key(), "true") + .option(CoreOptions.FILE_FORMAT.key(), "csv") + .column("id", DataTypes.INT()) + .column("year", DataTypes.INT()) + .column("month", DataTypes.INT()) + .partitionKeys("year", "month") + .build(), + false); + + FormatTable managedTable = (FormatTable) catalog.getTable(identifier); + BatchWriteBuilder writeBuilder = managedTable.newBatchWriteBuilder(); + try (BatchTableWrite write = writeBuilder.newWrite(); + BatchTableCommit commit = writeBuilder.newCommit()) { + write.write(GenericRow.of(1, 2024, 10)); + write.write(GenericRow.of(2, 2025, 10)); + write.write(GenericRow.of(3, 2025, 11)); + commit.commit(write.prepareCommit()); + } + + List> registeredPartitions = + Arrays.asList( + ImmutableMap.of("year", "2024", "month", "10"), + ImmutableMap.of("year", "2025", "month", "10"), + ImmutableMap.of("year", "2025", "month", "11")); + assertThat(catalog.listPartitions(identifier).stream().map(Partition::spec)) + .containsExactlyInAnyOrderElementsOf(registeredPartitions); + assertThat(managedTable.newReadBuilder().newScan().getClass().getSimpleName()) + .isEqualTo("ManagedFormatTableScan"); + + Map fileSystemOptions = new HashMap<>(managedTable.options()); + fileSystemOptions.put(METASTORE_PARTITIONED_TABLE.key(), "false"); + FormatTable fileSystemTable = + FormatTable.builder() + .fileIO(managedTable.fileIO()) + .identifier(Identifier.create("format_partition_db", "filesystem_scan")) + .rowType(managedTable.rowType()) + .partitionKeys(managedTable.partitionKeys()) + .location(managedTable.location()) + .format(managedTable.format()) + .options(fileSystemOptions) + .catalogContext(managedTable.catalogContext()) + .build(); + + Map partitionFilter = singletonMap("year", "2025"); + assertThat(read(managedTable, null, null, partitionFilter, null)) + .containsExactlyInAnyOrderElementsOf( + read(fileSystemTable, null, null, partitionFilter, null)); + } + @Test void testListPartitions() throws Exception { innerTestListPartitions(true); diff --git a/paimon-core/src/test/java/org/apache/paimon/table/FormatTableCompatibilityTest.java b/paimon-core/src/test/java/org/apache/paimon/table/FormatTableCompatibilityTest.java new file mode 100644 index 000000000000..29fe9f4fd7fe --- /dev/null +++ b/paimon-core/src/test/java/org/apache/paimon/table/FormatTableCompatibilityTest.java @@ -0,0 +1,185 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.table; + +import org.apache.paimon.catalog.CatalogContext; +import org.apache.paimon.catalog.Identifier; +import org.apache.paimon.fs.FileIO; +import org.apache.paimon.fs.local.LocalFileIO; +import org.apache.paimon.table.format.FormatTableCatalogProvider; +import org.apache.paimon.table.format.FormatTableCommit; +import org.apache.paimon.types.DataTypes; +import org.apache.paimon.types.RowType; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +import static org.apache.paimon.CoreOptions.FORMAT_TABLE_PARTITION_ONLY_VALUE_IN_PATH; +import static org.apache.paimon.CoreOptions.METASTORE_PARTITIONED_TABLE; +import static org.apache.paimon.CoreOptions.READ_BATCH_SIZE; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/** Compatibility tests for the public {@link FormatTable} API. */ +class FormatTableCompatibilityTest { + + @ParameterizedTest + @ValueSource(booleans = {true, false}) + void testCopyRejectsManagedPartitionStateChange(boolean managed) { + FormatTable table = formatTable(Boolean.toString(managed)); + + assertThatThrownBy( + () -> + table.copy( + Collections.singletonMap( + METASTORE_PARTITIONED_TABLE.key(), + Boolean.toString(!managed)))) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining(METASTORE_PARTITIONED_TABLE.key()); + } + + @Test + void testCopyAllowsSemanticallyEquivalentManagedPartitionOption() { + FormatTable table = formatTable("TRUE"); + + FormatTable copied = + table.copy(Collections.singletonMap(METASTORE_PARTITIONED_TABLE.key(), "true")); + + assertThat(copied).isNotSameAs(table); + assertThat(copied.options()) + .containsEntry(METASTORE_PARTITIONED_TABLE.key(), Boolean.TRUE.toString()); + } + + @Test + void testCopyRejectsChangingAbsentManagedPartitionDefault() { + FormatTable table = formatTable(Collections.emptyMap(), null); + + assertThat(table.options()).doesNotContainKey(METASTORE_PARTITIONED_TABLE.key()); + assertThatThrownBy( + () -> + table.copy( + Collections.singletonMap( + METASTORE_PARTITIONED_TABLE.key(), "true"))) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining(METASTORE_PARTITIONED_TABLE.key()); + } + + @ParameterizedTest + @ValueSource(booleans = {true, false}) + void testManagedCopyRejectsPhysicalPartitionLayoutChange(boolean onlyValueInPath) { + Map options = new LinkedHashMap<>(); + options.put(METASTORE_PARTITIONED_TABLE.key(), "true"); + options.put( + FORMAT_TABLE_PARTITION_ONLY_VALUE_IN_PATH.key(), Boolean.toString(onlyValueInPath)); + FormatTable table = formatTable(options, null); + + assertThatThrownBy( + () -> + table.copy( + Collections.singletonMap( + FORMAT_TABLE_PARTITION_ONLY_VALUE_IN_PATH.key(), + Boolean.toString(!onlyValueInPath)))) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining(FORMAT_TABLE_PARTITION_ONLY_VALUE_IN_PATH.key()); + } + + @ParameterizedTest + @ValueSource(booleans = {true, false}) + void testCopyPreservesManagedStateAndCatalogProviderForUnrelatedOption(boolean managed) { + Identifier identifier = Identifier.create("managed_db", "managed_table"); + FormatTableCatalogProvider catalogProvider = + new FormatTableCatalogProvider(identifier, () -> null); + Map options = new LinkedHashMap<>(); + options.put(METASTORE_PARTITIONED_TABLE.key(), Boolean.toString(managed)); + FormatTable table = formatTable(options, catalogProvider); + + FormatTable copied = table.copy(Collections.singletonMap(READ_BATCH_SIZE.key(), "128")); + + assertThat(copied).isNotSameAs(table); + assertThat(copied.options()) + .containsEntry(READ_BATCH_SIZE.key(), "128") + .containsEntry(METASTORE_PARTITIONED_TABLE.key(), Boolean.toString(managed)); + assertThat( + new org.apache.paimon.CoreOptions(copied.options()) + .partitionedTableInMetastore()) + .isEqualTo(managed); + assertThat(copied.catalogProvider()).isSameAs(catalogProvider); + assertThat(table.options()) + .containsExactlyEntriesOf(options) + .doesNotContainKey(READ_BATCH_SIZE.key()); + } + + @Test + void testManagedCatalogExtensionKeepsExistingImplementationsCompatible() throws Exception { + assertThat(FormatTable.class.getMethod("catalogProvider").isDefault()).isTrue(); + assertThat( + FormatTable.FormatTableImpl.class.getConstructor( + FileIO.class, + Identifier.class, + RowType.class, + List.class, + String.class, + FormatTable.Format.class, + Map.class, + String.class, + CatalogContext.class)) + .isNotNull(); + assertThat( + FormatTableCommit.class.getConstructor( + String.class, + List.class, + FileIO.class, + boolean.class, + boolean.class, + Identifier.class, + Map.class, + String.class, + CatalogContext.class)) + .isNotNull(); + } + + private static FormatTable formatTable(String managed) { + return formatTable( + Collections.singletonMap(METASTORE_PARTITIONED_TABLE.key(), managed), null); + } + + private static FormatTable formatTable( + Map options, FormatTableCatalogProvider catalogProvider) { + return FormatTable.builder() + .fileIO(LocalFileIO.create()) + .identifier(Identifier.create("managed_db", "managed_table")) + .rowType( + RowType.builder() + .field("id", DataTypes.INT()) + .field("dt", DataTypes.STRING()) + .build()) + .partitionKeys(Collections.singletonList("dt")) + .location("file:///warehouse/managed_table") + .format(FormatTable.Format.PARQUET) + .options(options) + .catalogProvider(catalogProvider) + .build(); + } +} diff --git a/paimon-core/src/test/java/org/apache/paimon/table/format/FormatTableCommitTest.java b/paimon-core/src/test/java/org/apache/paimon/table/format/FormatTableCommitTest.java new file mode 100644 index 000000000000..2bc74ca99bd6 --- /dev/null +++ b/paimon-core/src/test/java/org/apache/paimon/table/format/FormatTableCommitTest.java @@ -0,0 +1,247 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.table.format; + +import org.apache.paimon.catalog.Identifier; +import org.apache.paimon.fs.Path; +import org.apache.paimon.fs.RenamingTwoPhaseOutputStream; +import org.apache.paimon.fs.TwoPhaseOutputStream; +import org.apache.paimon.fs.local.LocalFileIO; +import org.apache.paimon.table.sink.CommitMessage; +import org.apache.paimon.utils.PartitionPathUtils; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.mockito.ArgumentCaptor; + +import java.io.IOException; +import java.util.Arrays; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.assertj.core.api.Assertions.entry; +import static org.mockito.ArgumentMatchers.anyList; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; + +/** Tests for {@link FormatTableCommit}. */ +class FormatTableCommitTest { + + @TempDir java.nio.file.Path tempDir; + + @Test + void testPartitionRegistrationFailurePreservesCommittedFiles() throws Exception { + LocalFileIO fileIO = LocalFileIO.create(); + Path tablePath = new Path(tempDir.toUri()); + Path targetPath = new Path(tablePath, "year=2025/month=10/data-1.csv"); + RenamingTwoPhaseOutputStream outputStream = + new RenamingTwoPhaseOutputStream(fileIO, targetPath, false); + outputStream.write(1); + TwoPhaseOutputStream.Committer committer = outputStream.closeForCommit(); + FormatTableCatalogProvider catalogProvider = mock(FormatTableCatalogProvider.class); + RuntimeException registrationFailure = + new RuntimeException("Catalog partition registration unavailable"); + doThrow(registrationFailure).when(catalogProvider).createPartitions(anyList()); + + FormatTableCommit commit = + new FormatTableCommit( + tablePath.toString(), + Arrays.asList("year", "month"), + fileIO, + false, + false, + Identifier.create("managed_db", "managed_table"), + null, + null, + null, + catalogProvider); + CommitMessage message = new TwoPhaseCommitMessage(committer); + + assertThatThrownBy(() -> commit.commit(Collections.singletonList(message))) + .isInstanceOf(RuntimeException.class) + .hasMessageContaining("Committed data files were preserved") + .hasMessageContaining("managed_db.managed_table") + .hasMessageContaining("MSCK REPAIR TABLE") + .hasRootCauseMessage("Catalog partition registration unavailable"); + + assertThat(fileIO.exists(targetPath)).isTrue(); + verify(catalogProvider).createPartitions(anyList()); + + // Flink calls abort on the same commit object after a failed commit. + commit.abort(Collections.singletonList(message)); + assertThat(fileIO.exists(targetPath)).isTrue(); + } + + @Test + void testFileCommitFailureStillDiscardsUncommittedFiles() throws Exception { + LocalFileIO fileIO = LocalFileIO.create(); + Path tablePath = new Path(tempDir.toUri()); + TwoPhaseOutputStream.Committer committer = mock(TwoPhaseOutputStream.Committer.class); + doThrow(new IOException("data commit failed")).when(committer).commit(fileIO); + FormatTableCatalogProvider catalogProvider = mock(FormatTableCatalogProvider.class); + FormatTableCommit commit = + new FormatTableCommit( + tablePath.toString(), + Arrays.asList("year", "month"), + fileIO, + false, + false, + Identifier.create("managed_db", "managed_table"), + null, + null, + null, + catalogProvider); + CommitMessage message = new TwoPhaseCommitMessage(committer); + + assertThatThrownBy(() -> commit.commit(Collections.singletonList(message))) + .isInstanceOf(RuntimeException.class) + .hasRootCauseMessage("data commit failed"); + + verify(committer).discard(fileIO); + verify(catalogProvider, never()).createPartitions(anyList()); + } + + @Test + void testRegistersRawPartitionValuesForEscapedPath() throws Exception { + Path tablePath = new Path(tempDir.toUri()); + LinkedHashMap rawSpec = new LinkedHashMap<>(); + rawSpec.put("year", "2025"); + rawSpec.put("month", "a b:c"); + // The writer escapes partition values when building the directory layout. + String partitionDir = PartitionPathUtils.generatePartitionPathUtil(rawSpec, false); + assertThat(partitionDir).isEqualTo("year=2025/month=a b%3Ac/"); + + FormatTableCatalogProvider catalogProvider = + commitPartitionedFile(tablePath, false, partitionDir); + + // The catalog must receive RAW values; readers re-escape them when probing directories. + assertThat(registeredSpec(catalogProvider)) + .containsExactly(entry("year", "2025"), entry("month", "a b:c")); + } + + @Test + void testForeignKeyValueSegmentsInLocationDoNotLeakIntoSpec() throws Exception { + Path tablePath = new Path(new Path(tempDir.toUri()), "env=prod/warehouse/tbl"); + + FormatTableCatalogProvider catalogProvider = + commitPartitionedFile(tablePath, false, "year=2025/month=10"); + + assertThat(registeredSpec(catalogProvider)) + .containsExactly(entry("year", "2025"), entry("month", "10")); + } + + @Test + void testValueOnlyPathUnderForeignKeyValueSegmentRegistersRawValues() throws Exception { + Path tablePath = new Path(new Path(tempDir.toUri()), "env=prod/warehouse/tbl"); + LinkedHashMap rawSpec = new LinkedHashMap<>(); + rawSpec.put("year", "2025"); + rawSpec.put("month", "a:b"); + String partitionDir = PartitionPathUtils.generatePartitionPathUtil(rawSpec, true); + assertThat(partitionDir).isEqualTo("2025/a%3Ab/"); + + FormatTableCatalogProvider catalogProvider = + commitPartitionedFile(tablePath, true, partitionDir); + + assertThat(registeredSpec(catalogProvider)) + .containsExactly(entry("year", "2025"), entry("month", "a:b")); + } + + @Test + void testMismatchedPartitionKeyInPathFailsWithClearMessage() { + Path tablePath = new Path(tempDir.toUri()); + + assertThatThrownBy(() -> commitPartitionedFile(tablePath, false, "year=2025/day=10")) + .isInstanceOf(RuntimeException.class) + .hasRootCauseInstanceOf(IllegalArgumentException.class) + .rootCause() + .hasMessageContaining("declares partition key 'day'") + .hasMessageContaining("partition key 'month' was expected"); + } + + @Test + void testValueOnlyStaticPartitionCannotEscapeTableLocation() throws Exception { + LocalFileIO fileIO = LocalFileIO.create(); + Path parentPath = new Path(tempDir.toUri()); + Path tablePath = new Path(parentPath, "table"); + Path siblingPath = new Path(parentPath, "keep"); + fileIO.mkdirs(tablePath); + fileIO.mkdirs(siblingPath); + Map staticPartition = Collections.singletonMap("year", ".."); + FormatTableCommit commit = + new FormatTableCommit( + tablePath.toString(), + Collections.singletonList("year"), + fileIO, + true, + true, + Identifier.create("managed_db", "managed_table"), + staticPartition, + null, + null, + null); + + assertThatThrownBy(() -> commit.commit(Collections.emptyList())) + .isInstanceOf(RuntimeException.class) + .hasRootCauseInstanceOf(IllegalArgumentException.class) + .hasRootCauseMessage( + "Partition value '..' cannot be used as a partition path component."); + assertThat(fileIO.exists(tablePath)).isTrue(); + assertThat(fileIO.exists(siblingPath)).isTrue(); + } + + private FormatTableCatalogProvider commitPartitionedFile( + Path tableLocation, boolean onlyValueInPath, String partitionDir) throws Exception { + LocalFileIO fileIO = LocalFileIO.create(); + Path targetPath = new Path(new Path(tableLocation, partitionDir), "data-1.csv"); + RenamingTwoPhaseOutputStream outputStream = + new RenamingTwoPhaseOutputStream(fileIO, targetPath, false); + outputStream.write(1); + TwoPhaseOutputStream.Committer committer = outputStream.closeForCommit(); + FormatTableCatalogProvider catalogProvider = mock(FormatTableCatalogProvider.class); + FormatTableCommit commit = + new FormatTableCommit( + tableLocation.toString(), + Arrays.asList("year", "month"), + fileIO, + onlyValueInPath, + false, + Identifier.create("managed_db", "managed_table"), + null, + null, + null, + catalogProvider); + commit.commit(Collections.singletonList(new TwoPhaseCommitMessage(committer))); + return catalogProvider; + } + + @SuppressWarnings({"unchecked", "rawtypes"}) + private static Map registeredSpec(FormatTableCatalogProvider catalogProvider) { + ArgumentCaptor>> captor = + ArgumentCaptor.forClass((Class) List.class); + verify(catalogProvider).createPartitions(captor.capture()); + assertThat(captor.getValue()).hasSize(1); + return captor.getValue().get(0); + } +} diff --git a/paimon-core/src/test/java/org/apache/paimon/table/format/ManagedFormatTableScanTest.java b/paimon-core/src/test/java/org/apache/paimon/table/format/ManagedFormatTableScanTest.java new file mode 100644 index 000000000000..0ca741386eed --- /dev/null +++ b/paimon-core/src/test/java/org/apache/paimon/table/format/ManagedFormatTableScanTest.java @@ -0,0 +1,553 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.table.format; + +import org.apache.paimon.PagedList; +import org.apache.paimon.catalog.Catalog; +import org.apache.paimon.catalog.Identifier; +import org.apache.paimon.data.BinaryString; +import org.apache.paimon.fs.FileStatus; +import org.apache.paimon.fs.Path; +import org.apache.paimon.fs.PositionOutputStream; +import org.apache.paimon.fs.local.LocalFileIO; +import org.apache.paimon.manifest.PartitionEntry; +import org.apache.paimon.partition.Partition; +import org.apache.paimon.partition.PartitionPredicate; +import org.apache.paimon.predicate.Predicate; +import org.apache.paimon.predicate.PredicateBuilder; +import org.apache.paimon.table.FormatTable; +import org.apache.paimon.table.source.Split; +import org.apache.paimon.types.DataTypes; +import org.apache.paimon.types.RowType; +import org.apache.paimon.utils.PartitionPathUtils; + +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.io.FileNotFoundException; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +import static org.apache.paimon.CoreOptions.FORMAT_TABLE_PARTITION_ONLY_VALUE_IN_PATH; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.ArgumentMatchers.isNull; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +/** Tests for managed format table partition discovery. */ +class ManagedFormatTableScanTest { + + private static final Identifier IDENTIFIER = Identifier.create("managed_db", "managed_table"); + + @TempDir java.nio.file.Path tempDir; + + @Test + void testProviderRegistersLargeBatchesInBoundedRequests() throws Exception { + Catalog catalog = mock(Catalog.class); + FormatTableCatalogProvider provider = provider(catalog); + List> specs = new ArrayList<>(); + for (int index = 0; index < 2500; index++) { + specs.add(Collections.singletonMap("year", String.format("y%04d", index))); + } + + provider.createPartitions(specs); + + @SuppressWarnings({"unchecked", "rawtypes"}) + org.mockito.ArgumentCaptor>> batches = + (org.mockito.ArgumentCaptor) org.mockito.ArgumentCaptor.forClass(List.class); + verify(catalog, times(3)).createPartitions(eq(IDENTIFIER), batches.capture()); + assertThat(batches.getAllValues().get(0)).hasSize(1000); + assertThat(batches.getAllValues().get(1)).hasSize(1000); + assertThat(batches.getAllValues().get(2)).hasSize(500); + assertThat( + batches.getAllValues().stream() + .flatMap(List::stream) + .collect(java.util.stream.Collectors.toList())) + .containsExactlyElementsOf(specs); + } + + @Test + void testProviderPaginatesAndCachesMatchingPattern() throws Exception { + Catalog catalog = mock(Catalog.class); + Partition october = partition("2025", "10"); + Partition november = partition("2025", "11"); + when(catalog.listPartitionsPaged(eq(IDENTIFIER), eq(1000), isNull(), eq("year=2025/%"))) + .thenReturn(new PagedList<>(Collections.singletonList(october), "next")); + when(catalog.listPartitionsPaged(eq(IDENTIFIER), eq(1000), eq("next"), eq("year=2025/%"))) + .thenReturn(new PagedList<>(Collections.singletonList(november), null)); + + FormatTableCatalogProvider provider = provider(catalog); + + assertThat(provider.listPartitions("year=2025/%")).containsExactly(october, november); + assertThat(provider.listPartitions("year=2025/%")).containsExactly(october, november); + verify(catalog, times(1)).listPartitionsPaged(IDENTIFIER, 1000, null, "year=2025/%"); + verify(catalog, times(1)).listPartitionsPaged(IDENTIFIER, 1000, "next", "year=2025/%"); + } + + @Test + void testProviderRejectsRepeatedPageTokenCycle() throws Exception { + Catalog catalog = mock(Catalog.class); + when(catalog.listPartitionsPaged(eq(IDENTIFIER), eq(1000), isNull(), isNull())) + .thenReturn(new PagedList<>(Collections.emptyList(), "a")); + when(catalog.listPartitionsPaged(eq(IDENTIFIER), eq(1000), eq("a"), isNull())) + .thenReturn(new PagedList<>(Collections.emptyList(), "b")) + .thenThrow(new AssertionError("provider requested page token 'a' twice")); + when(catalog.listPartitionsPaged(eq(IDENTIFIER), eq(1000), eq("b"), isNull())) + .thenReturn(new PagedList<>(Collections.emptyList(), "a")); + + assertThatThrownBy(() -> provider(catalog).listPartitions(null)) + .isInstanceOf(IllegalStateException.class) + .hasMessageContaining("repeated partition page token 'a'") + .hasMessageContaining(IDENTIFIER.getFullName()); + } + + @Test + void testCreatePartitionsInvalidatesCachedPattern() throws Exception { + Catalog catalog = mock(Catalog.class); + Partition oldPartition = partition("2025", "10"); + Partition newPartition = partition("2025", "11"); + when(catalog.listPartitionsPaged(eq(IDENTIFIER), eq(1000), isNull(), eq("year=2025/%"))) + .thenReturn( + new PagedList<>(Collections.singletonList(oldPartition), null), + new PagedList<>(Arrays.asList(oldPartition, newPartition), null)); + + FormatTableCatalogProvider provider = provider(catalog); + assertThat(provider.listPartitions("year=2025/%")).containsExactly(oldPartition); + assertThat(provider.listPartitions("year=2025/%")).containsExactly(oldPartition); + + List> created = Collections.singletonList(newPartition.spec()); + provider.createPartitions(created); + + assertThat(provider.listPartitions("year=2025/%")) + .containsExactly(oldPartition, newPartition); + verify(catalog).createPartitions(IDENTIFIER, created); + verify(catalog, times(2)).listPartitionsPaged(IDENTIFIER, 1000, null, "year=2025/%"); + } + + @Test + void testCreatePartitionsInvalidatesOtherProviderInSameProcess() throws Exception { + Catalog catalog = mock(Catalog.class); + Partition oldPartition = partition("2025", "10"); + Partition newPartition = partition("2025", "11"); + when(catalog.listPartitionsPaged(eq(IDENTIFIER), eq(1000), isNull(), eq("year=2025/%"))) + .thenReturn( + new PagedList<>(Collections.singletonList(oldPartition), null), + new PagedList<>(Arrays.asList(oldPartition, newPartition), null)); + FormatTableCatalogProvider readerProvider = provider(catalog); + FormatTableCatalogProvider writerProvider = provider(catalog); + + assertThat(readerProvider.listPartitions("year=2025/%")).containsExactly(oldPartition); + writerProvider.createPartitions(Collections.singletonList(newPartition.spec())); + + assertThat(readerProvider.listPartitions("year=2025/%")) + .containsExactly(oldPartition, newPartition); + verify(catalog, times(2)).listPartitionsPaged(IDENTIFIER, 1000, null, "year=2025/%"); + } + + @Test + void testCreatePartitionsFailureInvalidatesOtherProviderCache() throws Exception { + Catalog catalog = mock(Catalog.class); + Partition oldPartition = partition("2025", "10"); + Partition newPartition = partition("2025", "11"); + List storedPartitions = new ArrayList<>(); + storedPartitions.add(oldPartition); + when(catalog.listPartitionsPaged(eq(IDENTIFIER), eq(1000), isNull(), eq("year=2025/%"))) + .thenAnswer(ignored -> new PagedList<>(new ArrayList<>(storedPartitions), null)); + List> created = Collections.singletonList(newPartition.spec()); + RuntimeException responseLost = new RuntimeException("response lost"); + doAnswer( + ignored -> { + storedPartitions.add(newPartition); + throw responseLost; + }) + .when(catalog) + .createPartitions(IDENTIFIER, created); + FormatTableCatalogProvider readerProvider = provider(catalog); + FormatTableCatalogProvider writerProvider = provider(catalog); + + assertThat(readerProvider.listPartitions("year=2025/%")).containsExactly(oldPartition); + assertThatThrownBy(() -> writerProvider.createPartitions(created)).isSameAs(responseLost); + + assertThat(readerProvider.listPartitions("year=2025/%")) + .containsExactly(oldPartition, newPartition); + verify(catalog, times(2)).listPartitionsPaged(IDENTIFIER, 1000, null, "year=2025/%"); + } + + @Test + void testAdvanceGenerationInvalidatesProvidersInSameProcess() throws Exception { + Catalog catalog = mock(Catalog.class); + Partition oldPartition = partition("2025", "10"); + when(catalog.listPartitionsPaged(eq(IDENTIFIER), eq(1000), isNull(), eq("year=2025/%"))) + .thenReturn( + new PagedList<>(Arrays.asList(oldPartition, partition("2025", "11")), null), + new PagedList<>(Collections.singletonList(oldPartition), null)); + FormatTableCatalogProvider provider = provider(catalog); + + assertThat(provider.listPartitions("year=2025/%")).hasSize(2); + // The catalog partition DDL path (e.g. RESTCatalog.dropPartitions) advances the + // generation so same-process scans read their own writes within the cache TTL. + FormatTableCatalogProvider.advanceGeneration(IDENTIFIER); + + assertThat(provider.listPartitions("year=2025/%")).containsExactly(oldPartition); + verify(catalog, times(2)).listPartitionsPaged(IDENTIFIER, 1000, null, "year=2025/%"); + } + + @Test + void testLeadingPatternResidualFilterAndUnregisteredDirectory() throws Exception { + Catalog catalog = mock(Catalog.class); + when(catalog.listPartitionsPaged(eq(IDENTIFIER), eq(1000), isNull(), eq("year=2025/%"))) + .thenReturn( + new PagedList<>( + Arrays.asList(partition("2025", "10"), partition("2025", "11")), + null)); + TrackingLocalFileIO fileIO = new TrackingLocalFileIO(); + Path tablePath = new Path(tempDir.toUri()); + Path octoberFile = writeDataFile(fileIO, tablePath, "year=2025/month=10"); + Path novemberFile = writeDataFile(fileIO, tablePath, "year=2025/month=11"); + Path unregisteredFile = writeDataFile(fileIO, tablePath, "year=2025/month=12"); + FormatTable table = createTable(fileIO, tablePath, provider(catalog), false); + PredicateBuilder builder = new PredicateBuilder(table.partitionType()); + Predicate predicate = + PredicateBuilder.and(builder.equal(0, 2025), builder.greaterThan(1, 10)); + PartitionPredicate filter = + PartitionPredicate.fromPredicate(table.partitionType(), predicate); + + ManagedFormatTableScan scan = new ManagedFormatTableScan(table, filter, null); + List plannedFiles = plannedFiles(scan.plan().splits()); + + assertThat(scan.createPartitionNamePattern()).isEqualTo("year=2025/%"); + assertThat(plannedFiles).containsExactly(novemberFile); + assertThat(plannedFiles).doesNotContain(octoberFile, unregisteredFile); + assertThat(fileIO.listedPaths).containsExactly(new Path(tablePath, "year=2025/month=11")); + } + + @Test + void testListPartitionEntriesUsesCatalogVisibility() throws Exception { + Catalog catalog = mock(Catalog.class); + Partition catalogPartition = + new Partition(partition("2025", "11").spec(), 11L, 22L, 3L, 44L, 5, false); + when(catalog.listPartitionsPaged(eq(IDENTIFIER), eq(1000), isNull(), isNull())) + .thenReturn(new PagedList<>(Collections.singletonList(catalogPartition), null)); + TrackingLocalFileIO fileIO = new TrackingLocalFileIO(); + Path tablePath = new Path(tempDir.toUri()); + writeDataFile(fileIO, tablePath, "year=2025/month=11"); + writeDataFile(fileIO, tablePath, "year=2025/month=12"); + FormatTable table = createTable(fileIO, tablePath, provider(catalog), false); + + List entries = + new ManagedFormatTableScan(table, null, null).listPartitionEntries(); + + assertThat(entries) + .extracting( + entry -> entry.partition().getInt(0), entry -> entry.partition().getInt(1)) + .containsExactly(org.assertj.core.groups.Tuple.tuple(2025, 11)); + assertThat(entries.get(0).recordCount()).isEqualTo(11L); + assertThat(entries.get(0).fileSizeInBytes()).isEqualTo(22L); + assertThat(entries.get(0).fileCount()).isEqualTo(3L); + assertThat(entries.get(0).lastFileCreationTime()).isEqualTo(44L); + assertThat(entries.get(0).totalBuckets()).isEqualTo(5); + assertThat(fileIO.listedPaths).isEmpty(); + assertThat(fileIO.statusListedPaths).isEmpty(); + } + + @Test + void testUnderscoreInPartitionNameRemainsLiteralPrefix() { + LocalFileIO fileIO = LocalFileIO.create(); + Path tablePath = new Path(tempDir.toUri()); + FormatTable table = + createStringPartitionTable(fileIO, tablePath, provider(mock(Catalog.class))); + Predicate predicate = + new PredicateBuilder(table.partitionType()) + .equal(0, BinaryString.fromString("a_b")); + PartitionPredicate filter = + PartitionPredicate.fromPredicate(table.partitionType(), predicate); + + ManagedFormatTableScan scan = new ManagedFormatTableScan(table, filter, null); + + assertThat(scan.createPartitionNamePattern()).isEqualTo("year=a_b/%"); + } + + @Test + void testValueOnlyPartitionPath() throws Exception { + Catalog catalog = mock(Catalog.class); + when(catalog.listPartitionsPaged(eq(IDENTIFIER), eq(1000), isNull(), isNull())) + .thenReturn( + new PagedList<>(Collections.singletonList(partition("2025", "11")), null)); + LocalFileIO fileIO = LocalFileIO.create(); + Path tablePath = new Path(tempDir.toUri()); + Path dataFile = writeDataFile(fileIO, tablePath, "2025/11"); + FormatTable table = createTable(fileIO, tablePath, provider(catalog), true); + + List plannedFiles = + plannedFiles(new ManagedFormatTableScan(table, null, null).plan().splits()); + + assertThat(plannedFiles).containsExactly(dataFile); + } + + @Test + void testDuplicateCatalogPartitionPlansSplitsOnce() throws Exception { + Catalog catalog = mock(Catalog.class); + when(catalog.listPartitionsPaged(eq(IDENTIFIER), eq(1000), isNull(), isNull())) + .thenReturn( + new PagedList<>( + Arrays.asList(partition("2025", "11"), partition("2025", "11")), + null)); + LocalFileIO fileIO = LocalFileIO.create(); + Path tablePath = new Path(tempDir.toUri()); + Path dataFile = writeDataFile(fileIO, tablePath, "year=2025/month=11"); + FormatTable table = createTable(fileIO, tablePath, provider(catalog), false); + + List plannedFiles = + plannedFiles(new ManagedFormatTableScan(table, null, null).plan().splits()); + + assertThat(plannedFiles).containsExactly(dataFile); + } + + @Test + void testWhitespacePartitionValueIsVisible() throws Exception { + Catalog catalog = mock(Catalog.class); + Partition whitespacePartition = partition(" ", "11"); + when(catalog.listPartitionsPaged(eq(IDENTIFIER), eq(1000), isNull(), isNull())) + .thenReturn(new PagedList<>(Collections.singletonList(whitespacePartition), null)); + LocalFileIO fileIO = LocalFileIO.create(); + Path tablePath = new Path(tempDir.toUri()); + Path dataFile = + writeDataFile( + fileIO, + tablePath, + PartitionPathUtils.generatePartitionPath( + new LinkedHashMap<>(whitespacePartition.spec()))); + FormatTable table = createStringPartitionTable(fileIO, tablePath, provider(catalog)); + + List plannedFiles = + plannedFiles(new ManagedFormatTableScan(table, null, null).plan().splits()); + + assertThat(plannedFiles).containsExactly(dataFile); + } + + @Test + void testEmptyPartitionValueReportsCorruptMetadata() throws Exception { + Catalog catalog = mock(Catalog.class); + Partition emptyValuePartition = partition("2025", ""); + when(catalog.listPartitionsPaged(eq(IDENTIFIER), eq(1000), isNull(), isNull())) + .thenReturn(new PagedList<>(Collections.singletonList(emptyValuePartition), null)); + LocalFileIO fileIO = LocalFileIO.create(); + Path tablePath = new Path(tempDir.toUri()); + FormatTable table = createStringPartitionTable(fileIO, tablePath, provider(catalog)); + + assertThatThrownBy(() -> new ManagedFormatTableScan(table, null, null).plan().splits()) + .isInstanceOf(IllegalStateException.class) + .hasMessageContaining("corrupt partition metadata") + .hasMessageContaining(IDENTIFIER.getFullName()); + } + + @Test + void testCatalogFailureDoesNotFallBackToFileSystem() throws Exception { + Catalog catalog = mock(Catalog.class); + RuntimeException catalogFailure = + new RuntimeException("Catalog partition listing unavailable"); + when(catalog.listPartitionsPaged(eq(IDENTIFIER), eq(1000), isNull(), isNull())) + .thenThrow(catalogFailure); + TrackingLocalFileIO fileIO = new TrackingLocalFileIO(); + Path tablePath = new Path(tempDir.toUri()); + writeDataFile(fileIO, tablePath, "year=2025/month=11"); + FormatTable table = createTable(fileIO, tablePath, provider(catalog), false); + + assertThatThrownBy(() -> new ManagedFormatTableScan(table, null, null).plan().splits()) + .isSameAs(catalogFailure); + assertThat(fileIO.listedPaths).isEmpty(); + } + + @Test + @DisplayName( + "treats a registered partition with a missing directory as an empty partition " + + "(requires real object-store validation)") + void testMissingManagedPartitionReadsAsEmpty() throws Exception { + Catalog catalog = mock(Catalog.class); + when(catalog.listPartitionsPaged(eq(IDENTIFIER), eq(1000), isNull(), isNull())) + .thenReturn( + new PagedList<>(Collections.singletonList(partition("2025", "11")), null)); + Path tablePath = new Path(tempDir.toUri()); + Path missingPath = new Path(tablePath, "year=2025/month=11"); + LocalFileIO fileIO = + new LocalFileIO() { + @Override + public FileStatus[] listFiles(Path path, boolean recursive) throws IOException { + assertThat(path).isEqualTo(missingPath); + throw new FileNotFoundException(path.toString()); + } + }; + FormatTable table = createTable(fileIO, tablePath, provider(catalog), false); + + // A registered partition whose directory is missing (e.g. ADD PARTITION before the first + // INSERT) must not fail the whole scan; it reads as an empty partition, matching Hive. + List plannedFiles = + plannedFiles(new ManagedFormatTableScan(table, null, null).plan().splits()); + assertThat(plannedFiles).isEmpty(); + } + + @Test + void testTraversalPartitionValueReportsCorruptMetadata() throws Exception { + Catalog catalog = mock(Catalog.class); + Partition traversalPartition = partition("2025", ".."); + when(catalog.listPartitionsPaged(eq(IDENTIFIER), eq(1000), isNull(), isNull())) + .thenReturn(new PagedList<>(Collections.singletonList(traversalPartition), null)); + LocalFileIO fileIO = LocalFileIO.create(); + Path tablePath = new Path(tempDir.toUri()); + FormatTable table = createStringPartitionTable(fileIO, tablePath, provider(catalog), true); + + assertThatThrownBy(() -> new ManagedFormatTableScan(table, null, null).plan().splits()) + .isInstanceOf(IllegalStateException.class) + .hasMessageContaining("corrupt partition metadata") + .hasMessageContaining(IDENTIFIER.getFullName()); + } + + @Test + void testDotDotValueInKeyValueLayoutRemainsVisible() throws Exception { + Catalog catalog = mock(Catalog.class); + Partition partition = partition("2025", ".."); + when(catalog.listPartitionsPaged(eq(IDENTIFIER), eq(1000), isNull(), isNull())) + .thenReturn(new PagedList<>(Collections.singletonList(partition), null)); + LocalFileIO fileIO = LocalFileIO.create(); + Path tablePath = new Path(tempDir.toUri()); + Path dataFile = writeDataFile(fileIO, tablePath, "year=2025/month=.."); + FormatTable table = createStringPartitionTable(fileIO, tablePath, provider(catalog)); + + List plannedFiles = + plannedFiles(new ManagedFormatTableScan(table, null, null).plan().splits()); + + assertThat(plannedFiles).containsExactly(dataFile); + } + + private FormatTableCatalogProvider provider(Catalog catalog) { + return new FormatTableCatalogProvider(IDENTIFIER, () -> catalog); + } + + private FormatTable createTable( + LocalFileIO fileIO, + Path tablePath, + FormatTableCatalogProvider provider, + boolean valueOnlyPath) { + RowType rowType = + RowType.builder() + .field("year", DataTypes.INT()) + .field("month", DataTypes.INT()) + .field("id", DataTypes.INT()) + .build(); + return FormatTable.builder() + .fileIO(fileIO) + .identifier(IDENTIFIER) + .rowType(rowType) + .partitionKeys(Arrays.asList("year", "month")) + .location(tablePath.toString()) + .format(FormatTable.Format.CSV) + .options( + Collections.singletonMap( + FORMAT_TABLE_PARTITION_ONLY_VALUE_IN_PATH.key(), + Boolean.toString(valueOnlyPath))) + .catalogProvider(provider) + .build(); + } + + private FormatTable createStringPartitionTable( + LocalFileIO fileIO, Path tablePath, FormatTableCatalogProvider provider) { + return createStringPartitionTable(fileIO, tablePath, provider, false); + } + + private FormatTable createStringPartitionTable( + LocalFileIO fileIO, + Path tablePath, + FormatTableCatalogProvider provider, + boolean valueOnlyPath) { + RowType rowType = + RowType.builder() + .field("year", DataTypes.STRING()) + .field("month", DataTypes.STRING()) + .field("id", DataTypes.INT()) + .build(); + return FormatTable.builder() + .fileIO(fileIO) + .identifier(IDENTIFIER) + .rowType(rowType) + .partitionKeys(Arrays.asList("year", "month")) + .location(tablePath.toString()) + .format(FormatTable.Format.CSV) + .options( + Collections.singletonMap( + FORMAT_TABLE_PARTITION_ONLY_VALUE_IN_PATH.key(), + Boolean.toString(valueOnlyPath))) + .catalogProvider(provider) + .build(); + } + + private Path writeDataFile(LocalFileIO fileIO, Path tablePath, String partitionPath) + throws IOException { + Path file = new Path(new Path(tablePath, partitionPath), "data.csv"); + fileIO.mkdirs(file.getParent()); + try (PositionOutputStream out = fileIO.newOutputStream(file, false)) { + out.write(1); + } + return file; + } + + private static Partition partition(String year, String month) { + Map spec = new LinkedHashMap<>(); + spec.put("year", year); + spec.put("month", month); + return new Partition(spec, 0, 0, 0, 0, -1, false); + } + + private static List plannedFiles(List splits) { + return splits.stream() + .map(FormatDataSplit.class::cast) + .flatMap(split -> split.files().stream()) + .map(FormatDataSplit.FileMeta::filePath) + .collect(Collectors.toList()); + } + + private static class TrackingLocalFileIO extends LocalFileIO { + + private final List listedPaths = new ArrayList<>(); + private final List statusListedPaths = new ArrayList<>(); + + @Override + public FileStatus[] listStatus(Path path) throws IOException { + statusListedPaths.add(path); + return super.listStatus(path); + } + + @Override + public FileStatus[] listFiles(Path path, boolean recursive) throws IOException { + listedPaths.add(path); + return super.listFiles(path, recursive); + } + } +} diff --git a/paimon-core/src/test/java/org/apache/paimon/utils/PartitionPathUtilsTest.java b/paimon-core/src/test/java/org/apache/paimon/utils/PartitionPathUtilsTest.java index a21b168f54ea..6b2952778003 100644 --- a/paimon-core/src/test/java/org/apache/paimon/utils/PartitionPathUtilsTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/utils/PartitionPathUtilsTest.java @@ -19,17 +19,25 @@ package org.apache.paimon.utils; import org.apache.paimon.data.GenericRow; +import org.apache.paimon.fs.Path; import org.apache.paimon.predicate.Predicate; import org.apache.paimon.predicate.PredicateBuilder; import org.apache.paimon.types.DataTypes; import org.apache.paimon.types.RowType; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +import java.util.Arrays; +import java.util.LinkedHashMap; import static org.apache.paimon.utils.PartitionPathUtils.mightMatch; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.assertj.core.api.Assertions.entry; -/** Tests for {@link PartitionPathUtils#mightMatch}. */ +/** Tests for {@link PartitionPathUtils}. */ class PartitionPathUtilsTest { private final RowType partitionType = @@ -39,6 +47,46 @@ class PartitionPathUtilsTest { .build(); private final PredicateBuilder builder = new PredicateBuilder(partitionType); + @ParameterizedTest + @ValueSource(strings = {".", ".."}) + void testValueOnlyDotSegmentIsRejected(String rawValue) { + LinkedHashMap partitionSpec = new LinkedHashMap<>(); + partitionSpec.put("pt", rawValue); + + assertThatThrownBy(() -> PartitionPathUtils.generatePartitionPathUtil(partitionSpec, true)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining(rawValue); + } + + @ParameterizedTest + @ValueSource(strings = {".", ".."}) + void testKeyedDotValueKeepsCompatibleSafeLayout(String rawValue) { + LinkedHashMap partitionSpec = new LinkedHashMap<>(); + partitionSpec.put("pt", rawValue); + + String partitionPath = PartitionPathUtils.generatePartitionPathUtil(partitionSpec, false); + Path resolvedPath = new Path(new Path("file:///warehouse/table"), partitionPath); + + assertThat(partitionPath).isEqualTo("pt=" + rawValue + Path.SEPARATOR); + assertThat(PartitionPathUtils.extractPartitionSpecFromPath(resolvedPath)) + .containsExactly(entry("pt", rawValue)); + } + + @Test + void testTrailingPartitionExtractionStopsAtTableBoundary() { + Path partitionPath = new Path("file:///warehouse/env=prod/table/dt=20260718/hh=10"); + + assertThat( + PartitionPathUtils.extractPartitionSpecFromPath( + partitionPath, Arrays.asList("dt", "hh"))) + .containsExactly(entry("dt", "20260718"), entry("hh", "10")); + assertThat( + PartitionPathUtils.extractPartitionSpecFromPath( + new Path("file:///warehouse/dt=parent/table/wrong=20260718/hh=10"), + Arrays.asList("dt", "hh"))) + .isNull(); + } + @Test void testNullPredicate() { assertThat(mightMatch(null, 0, 0, GenericRow.of(2024, 5))).isTrue(); diff --git a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/format/FormatTableBatchWriteBase.scala b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/format/FormatTableBatchWriteBase.scala index 853c7f41467b..bec591bbd7d1 100644 --- a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/format/FormatTableBatchWriteBase.scala +++ b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/format/FormatTableBatchWriteBase.scala @@ -49,6 +49,8 @@ abstract class FormatTableBatchWriteBase( extends Logging with Serializable { + @volatile protected var commitStarted: Boolean = false + protected val batchWriteBuilder: BatchWriteBuilder = { val builder = table.newBatchWriteBuilder() // todo: add test for static overwrite the whole table @@ -65,6 +67,7 @@ abstract class FormatTableBatchWriteBase( } protected def commitMessages(messages: Array[WriterCommitMessage]): Unit = { + commitStarted = true logInfo(s"Committing to FormatTable ${table.name()}") val batchTableCommit = batchWriteBuilder.newCommit() val commitMessages = WriteTaskResult.merge(messages).asJava @@ -80,6 +83,12 @@ abstract class FormatTableBatchWriteBase( } protected def abortMessages(messages: Array[WriterCommitMessage]): Unit = { + if (commitStarted) { + logWarning( + s"Skip abort cleanup for FormatTable ${table.name()} because commit has already started") + return + } + logInfo(s"Aborting write to FormatTable ${table.name()}") val batchTableCommit = batchWriteBuilder.newCommit() val commitMessages = WriteTaskResult.merge(messages).asJava diff --git a/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/format/FormatTableBatchWriteTest.scala b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/format/FormatTableBatchWriteTest.scala new file mode 100644 index 000000000000..ce4d1951e1b4 --- /dev/null +++ b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/format/FormatTableBatchWriteTest.scala @@ -0,0 +1,98 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.spark.format + +import org.apache.paimon.spark.write.FormatTableWriteTaskResult +import org.apache.paimon.table.FormatTable +import org.apache.paimon.table.sink.{BatchTableCommit, BatchWriteBuilder} + +import org.apache.spark.SparkFunSuite +import org.apache.spark.sql.connector.write.WriterCommitMessage +import org.apache.spark.sql.types.StructType + +import java.lang.reflect.{InvocationHandler, Method, Proxy} +import java.util.concurrent.atomic.AtomicInteger + +class FormatTableBatchWriteTest extends SparkFunSuite { + + test("abort after commit starts does not clean up potentially committed files") { + val commitCalls = new AtomicInteger + val abortCalls = new AtomicInteger + val batchWrite = + new FormatTableBatchWrite(formatTable(commitCalls, abortCalls), None, None, StructType(Nil)) + val messages: Array[WriterCommitMessage] = Array(FormatTableWriteTaskResult(Seq.empty)) + + val error = intercept[RuntimeException] { + batchWrite.commit(messages) + } + assert(error.getMessage == "partition registration response lost") + + // Spark invokes abort after commit throws. The commit outcome may be ambiguous, so cleanup + // could delete files which were already committed and registered remotely. + batchWrite.abort(messages) + + assert(commitCalls.get == 1) + assert(abortCalls.get == 0) + } + + test("abort before commit delegates cleanup") { + val commitCalls = new AtomicInteger + val abortCalls = new AtomicInteger + val batchWrite = + new FormatTableBatchWrite(formatTable(commitCalls, abortCalls), None, None, StructType(Nil)) + + batchWrite.abort(Array(FormatTableWriteTaskResult(Seq.empty))) + + assert(commitCalls.get == 0) + assert(abortCalls.get == 1) + } + + private def formatTable(commitCalls: AtomicInteger, abortCalls: AtomicInteger): FormatTable = { + val tableCommit = proxy(classOf[BatchTableCommit]) { + case "commit" => + commitCalls.incrementAndGet() + throw new RuntimeException("partition registration response lost") + case "abort" => + abortCalls.incrementAndGet() + null + } + val writeBuilder = proxy(classOf[BatchWriteBuilder]) { case "newCommit" => tableCommit } + proxy(classOf[FormatTable]) { + case "newBatchWriteBuilder" => writeBuilder + case "name" => "test_db.format_table" + } + } + + private def proxy[T](clazz: Class[T])(responses: PartialFunction[String, AnyRef]): T = { + Proxy + .newProxyInstance( + clazz.getClassLoader, + Array(clazz), + new InvocationHandler { + override def invoke(proxy: Any, method: Method, args: Array[AnyRef]): AnyRef = { + responses.applyOrElse( + method.getName, + (name: String) => + throw new UnsupportedOperationException(s"Unexpected $name invocation")) + } + } + ) + .asInstanceOf[T] + } +} From eb47e824789ae09fa9674c857e2d2c683c9c620d Mon Sep 17 00:00:00 2001 From: Sun Dapeng Date: Sun, 19 Jul 2026 15:16:34 +0800 Subject: [PATCH 2/3] [spark] Support managed Format Table partition DDL Route ALTER TABLE ADD/DROP PARTITION for catalog-managed Format Tables through a Spark-side partition gateway. ADD forwards the whole batch with the IF NOT EXISTS flag so the catalog can apply it atomically and creates partition directories; strict batch atomicity stays with the catalog service. DROP unregisters partitions before deleting their directories, expands partial specs through bounded catalog pagination, and never touches data that is not registered. Session-level overrides of the managed flag are ignored when classifying tables, and unmanaged Format Tables fail with an explicit unsupported error. --- docs/docs/spark/sql-ddl.md | 46 + .../org/apache/paimon/spark/SparkCatalog.java | 9 +- .../spark/catalog/FormatTableCatalog.java | 10 +- .../spark/PaimonPartitionManagement.scala | 288 +++++- .../PaimonFormatTablePartitionDdlExec.scala | 145 +++ .../spark/execution/PaimonStrategy.scala | 34 +- .../CatalogFormatTablePartitionCatalog.scala | 92 ++ .../format/FormatTablePartitionCatalog.scala | 42 + .../spark/format/PaimonFormatTable.scala | 10 + .../paimon/spark/util/OptionUtils.scala | 38 +- .../extensions/RewriteSparkDDLCommands.scala | 47 +- .../paimon/spark/util/OptionUtilsTest.scala | 118 +++ .../FormatTablePartitionDdlPlanningTest.scala | 960 ++++++++++++++++++ ...talogFormatTablePartitionCatalogTest.scala | 200 ++++ .../FormatTablePartitionManagementTest.scala | 738 ++++++++++++++ .../ManagedFormatTableCatalogTest.scala | 45 + 16 files changed, 2768 insertions(+), 54 deletions(-) create mode 100644 paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/execution/PaimonFormatTablePartitionDdlExec.scala create mode 100644 paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/format/CatalogFormatTablePartitionCatalog.scala create mode 100644 paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/format/FormatTablePartitionCatalog.scala create mode 100644 paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/util/OptionUtilsTest.scala create mode 100644 paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/execution/FormatTablePartitionDdlPlanningTest.scala create mode 100644 paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/format/CatalogFormatTablePartitionCatalogTest.scala create mode 100644 paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/format/FormatTablePartitionManagementTest.scala create mode 100644 paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/format/ManagedFormatTableCatalogTest.scala diff --git a/docs/docs/spark/sql-ddl.md b/docs/docs/spark/sql-ddl.md index aa9310b8b478..4bc2bfcfbf26 100644 --- a/docs/docs/spark/sql-ddl.md +++ b/docs/docs/spark/sql-ddl.md @@ -206,6 +206,52 @@ CREATE TABLE my_table ( ); ``` +### Manage Format Table Partitions + +For managed Format Tables, whose `metastore.partitioned-table` option is `true`, partition +metadata is stored in the catalog and Spark supports the standard partition DDL: + +```sql +ALTER TABLE my_table ADD PARTITION (dt='2025-01-01'); +ALTER TABLE my_table DROP PARTITION (dt='2025-01-01'); +MSCK REPAIR TABLE my_table; +SHOW PARTITIONS my_table; +``` + +On a Format Table that is not managed, `ADD PARTITION`, `DROP PARTITION` and +`MSCK REPAIR TABLE` fail with an error. + +`SHOW PARTITIONS` throws an error when the number of partitions exceeds +`spark.paimon.format-table.show-partitions.max-results` (default: 10000). For larger tables, use +`CALL sys.list_format_table_partitions` to list partitions page by page. + +`ADD PARTITION` creates the partition directory and registers the partition; querying a newly +added partition before any data is written returns no rows. `DROP PARTITION` unregisters the +partition and deletes its directory. + +:::info + +`metastore.partitioned-table = true` enables catalog-managed partitions, which requires an +internal Format Table in a catalog that supports it (currently the REST catalog) and cannot be +combined with `format-table.implementation = engine`. The REST catalog validates this +combination on `CREATE TABLE`, with catalog-level table defaults +(`spark.sql.catalog.paimon.table-default.*`) participating in the effective options: a default +that makes the combination invalid fails the DDL. Other catalogs treat the option as inert. + +A table that carries `metastore.partitioned-table = true` where it cannot be honored (a non-REST +catalog, or an external table) loads as an unmanaged Format Table and a warning is logged. To +remove the option, use `ALTER TABLE my_table UNSET TBLPROPERTIES ('metastore.partitioned-table')`. +On a REST catalog, an existing Format Table whose partitions were never registered reads as empty +until you register them with `MSCK REPAIR TABLE my_table` or +`CALL sys.sync_format_table_metadata`. + +Mixed-version note: only writers that support catalog-managed partitions register the partitions +they produce. During a rolling upgrade, upgrade all writers before relying on managed reads, or +run `MSCK REPAIR TABLE` / `CALL sys.sync_format_table_metadata` afterwards, since data written by +an older writer is not visible to a managed read until its partitions are registered. + +::: + ### Create External Table When the catalog's `metastore` type is `hive`, if the `location` is specified when creating a table, that table will be considered an external table; otherwise, it will be a managed table. diff --git a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/SparkCatalog.java b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/SparkCatalog.java index f248ee70ae2d..f69c9083f84a 100644 --- a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/SparkCatalog.java +++ b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/SparkCatalog.java @@ -36,6 +36,8 @@ import org.apache.paimon.spark.catalog.SupportView; import org.apache.paimon.spark.catalog.functions.FunctionIdentifierConverter; import org.apache.paimon.spark.catalog.functions.PaimonFunctions; +import org.apache.paimon.spark.format.CatalogFormatTablePartitionCatalog; +import org.apache.paimon.spark.format.FormatTablePartitionCatalog; import org.apache.paimon.spark.utils.CatalogUtils; import org.apache.paimon.table.FormatTable; import org.apache.paimon.table.iceberg.IcebergTable; @@ -805,7 +807,12 @@ protected org.apache.spark.sql.connector.catalog.Table loadSparkTable( copyWithSQLConf( catalog.getTable(tblIdent), catalogName, tblIdent, extraOptions); if (table instanceof FormatTable) { - return toSparkFormatTable(ident, (FormatTable) table); + FormatTable formatTable = (FormatTable) table; + FormatTablePartitionCatalog partitionCatalog = + formatTable.catalogProvider() != null + ? new CatalogFormatTablePartitionCatalog(catalog, tblIdent) + : null; + return toSparkFormatTable(ident, formatTable, partitionCatalog); } else if (table instanceof IcebergTable) { return new SparkIcebergTable(table); } else if (table instanceof LanceTable) { diff --git a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/catalog/FormatTableCatalog.java b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/catalog/FormatTableCatalog.java index 3893e670685b..1493108729e1 100644 --- a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/catalog/FormatTableCatalog.java +++ b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/catalog/FormatTableCatalog.java @@ -24,6 +24,7 @@ import org.apache.paimon.options.Options; import org.apache.paimon.spark.SparkSource; import org.apache.paimon.spark.SparkTypeUtils; +import org.apache.paimon.spark.format.FormatTablePartitionCatalog; import org.apache.paimon.spark.format.PaimonFormatTable; import org.apache.paimon.table.FormatTable; import org.apache.paimon.utils.TypeUtils; @@ -60,10 +61,17 @@ default boolean isFormatTable(@Nullable String provide) { } default Table toSparkFormatTable(Identifier ident, FormatTable formatTable) { + return toSparkFormatTable(ident, formatTable, null); + } + + default Table toSparkFormatTable( + Identifier ident, + FormatTable formatTable, + @Nullable FormatTablePartitionCatalog partitionCatalog) { Map optionsMap = formatTable.options(); CoreOptions coreOptions = new CoreOptions(optionsMap); if (coreOptions.formatTableImplementationIsPaimon()) { - return new PaimonFormatTable(formatTable); + return new PaimonFormatTable(formatTable, partitionCatalog); } else { SparkSession spark = PaimonSparkSession$.MODULE$.active(); List pathList = new ArrayList<>(); diff --git a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/PaimonPartitionManagement.scala b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/PaimonPartitionManagement.scala index cd5955b0ff39..c25cf20063be 100644 --- a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/PaimonPartitionManagement.scala +++ b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/PaimonPartitionManagement.scala @@ -19,11 +19,13 @@ package org.apache.paimon.spark import org.apache.paimon.CoreOptions +import org.apache.paimon.fs.Path import org.apache.paimon.partition.PartitionStatistics -import org.apache.paimon.table.{FileStoreTable, Table} +import org.apache.paimon.spark.format.FormatTablePartitionCatalog +import org.apache.paimon.table.{FileStoreTable, FormatTable, Table} import org.apache.paimon.table.source.ScanMode import org.apache.paimon.types.RowType -import org.apache.paimon.utils.{InternalRowPartitionComputer, TypeUtils} +import org.apache.paimon.utils.{InternalRowPartitionComputer, PartitionPathUtils, TypeUtils} import org.apache.spark.internal.Logging import org.apache.spark.sql.Row @@ -35,58 +37,43 @@ import org.apache.spark.sql.types.StructType import java.util.{Map => JMap, Objects} import scala.collection.JavaConverters._ +import scala.collection.mutable.{ArrayBuffer, HashSet} trait PaimonPartitionManagement extends SupportsAtomicPartitionManagement with Logging { val table: Table + def partitionCatalog: FormatTablePartitionCatalog = null + lazy val partitionRowType: RowType = TypeUtils.project(table.rowType, table.partitionKeys) override lazy val partitionSchema: StructType = SparkTypeUtils.fromPaimonRowType(partitionRowType) private def toPaimonPartitions(rows: Array[InternalRow]): Array[java.util.Map[String, String]] = { - table match { - case fileStoreTable: FileStoreTable => - val partitionKeys = table.partitionKeys().asScala.toSeq - val partitionDefaultName = fileStoreTable.coreOptions().partitionDefaultName() - val legacyPartitionName = CoreOptions.fromMap(table.options()).legacyPartitionName - - rows.map { - r => - val partitionFieldCount = r.numFields - require( - partitionFieldCount <= partitionKeys.length, - s"Partition values length $partitionFieldCount exceeds partition keys " + - s"${partitionKeys.mkString("[", ", ", "]")}." - ) - val partitionNames = partitionKeys.take(partitionFieldCount) - val currentPartitionRowType = - if (partitionFieldCount == partitionRowType.getFieldCount) { - partitionRowType - } else { - TypeUtils.project(table.rowType, partitionNames.asJava) - } - val currentPartitionSchema = - if (partitionFieldCount == partitionSchema.length) { - partitionSchema - } else { - SparkTypeUtils.fromPaimonRowType(currentPartitionRowType) - } - val rowConverter = CatalystTypeConverters.createToScalaConverter( - CharVarcharUtils.replaceCharVarcharWithString(currentPartitionSchema)) - val rowDataPartitionComputer = new InternalRowPartitionComputer( - partitionDefaultName, - currentPartitionRowType, - partitionNames.toArray, - legacyPartitionName - ) - - rowDataPartitionComputer.generatePartValues( - new SparkRow(currentPartitionRowType, rowConverter(r).asInstanceOf[Row])) - } - case _ => - throw new UnsupportedOperationException("Only FileStoreTable supports partitions.") - } + val partitionKeys = table.partitionKeys().asScala.toSeq + + rows.map(r => toPaimonPartition(r, partitionKeys.take(r.numFields))) + } + + private def toPaimonPartition( + row: InternalRow, + partitionNames: Seq[String]): java.util.Map[String, String] = { + val coreOptions = CoreOptions.fromMap(table.options()) + val partitionDefaultName = coreOptions.partitionDefaultName() + val legacyPartitionName = coreOptions.legacyPartitionName + val currentPartitionRowType = TypeUtils.project(table.rowType, partitionNames.asJava) + val currentPartitionSchema = SparkTypeUtils.fromPaimonRowType(currentPartitionRowType) + val rowConverter = CatalystTypeConverters.createToScalaConverter( + CharVarcharUtils.replaceCharVarcharWithString(currentPartitionSchema)) + val rowDataPartitionComputer = new InternalRowPartitionComputer( + partitionDefaultName, + currentPartitionRowType, + partitionNames.toArray, + legacyPartitionName + ) + + rowDataPartitionComputer.generatePartValues( + new SparkRow(currentPartitionRowType, rowConverter(row).asInstanceOf[Row])) } override def dropPartitions(rows: Array[InternalRow]): Boolean = { @@ -116,6 +103,142 @@ trait PaimonPartitionManagement extends SupportsAtomicPartitionManagement with L } } + /** + * Resolves, with a single catalog list-by-names lookup, which of the given complete partition + * specs are registered for a managed Format Table. The result is aligned with the input arrays. + */ + private[spark] def formatTablePartitionsRegistered( + partitionNames: Array[Array[String]], + rows: Array[InternalRow]): Array[Boolean] = { + if (rows.isEmpty) { + return Array.empty + } + + table match { + case formatTable: FormatTable + if CoreOptions.fromMap(formatTable.options()).partitionedTableInMetastore() => + val requested = + rows.zip(partitionNames).map { case (row, names) => toPaimonPartition(row, names.toSeq) } + val registered = requirePartitionCatalog().listPartitionsByNames(requested.toSeq.asJava) + val registeredSpecs = registered.asScala.map(_.asScala.toMap).toSet + requested.map(spec => registeredSpecs.contains(spec.asScala.toMap)) + case _ => + throw new UnsupportedOperationException( + "Only managed Format Table supports partition registration lookup.") + } + } + + /** + * Drops the given partitions: complete specs are unregistered and their directories deleted + * as-is, partial specs are expanded to the registered leaf partitions they cover. Callers are + * responsible for resolving which complete specs are actually registered first (see + * [[formatTablePartitionsRegistered]]), so unregistered data directories are never deleted. + */ + private[spark] def dropFormatTablePartitions( + partitionNames: Array[Array[String]], + rows: Array[InternalRow]): Boolean = { + table match { + case formatTable: FormatTable + if CoreOptions.fromMap(formatTable.options()).partitionedTableInMetastore() => + val partitionKeyCount = formatTable.partitionKeys().size() + val requested = + rows.zip(partitionNames).map { case (row, names) => toPaimonPartition(row, names.toSeq) } + val partitions = ArrayBuffer.empty[java.util.Map[String, String]] + val seenPartitions = HashSet.empty[Map[String, String]] + + def addPartition(partition: java.util.Map[String, String]): Unit = { + if (seenPartitions.add(partition.asScala.toMap)) { + partitions += partition + } + } + + // Preserve exact requests as-is and let discovery add only missing complete leaves. + requested.filter(_.size() == partitionKeyCount).foreach(addPartition) + val partialSpecs = requested.filter(_.size() < partitionKeyCount).toSeq.distinct + if (partialSpecs.nonEmpty) { + def matchesRequestedPartial(partition: java.util.Map[String, String]): Boolean = { + partialSpecs.exists(_.asScala.forall { + case (key, value) => Objects.equals(value, partition.get(key)) + }) + } + + // One unfiltered paged traversal resolves every partial spec; the requested constraints + // are enforced client-side. + val catalog = requirePartitionCatalog() + var pageToken: String = null + do { + val page = catalog.listPartitions( + new java.util.LinkedHashMap[String, String](), + pageToken, + PaimonPartitionManagement.ManagedPartitionDropPageSize) + page.partitions.asScala.foreach { + partition => + val validated = validateManagedDropPartition(formatTable, partition) + if (matchesRequestedPartial(validated)) { + addPartition(validated) + } + } + pageToken = Option(page.nextPageToken).filter(_.nonEmpty).orNull + } while (pageToken != null) + } + dropManagedFormatTablePartitions(formatTable, partitions.toSeq) + case _ => + throw new UnsupportedOperationException( + "Only managed Format Table supports named partition drop.") + } + } + + private def dropManagedFormatTablePartitions( + formatTable: FormatTable, + partitions: Seq[java.util.Map[String, String]]): Boolean = { + // Unregister first so new queries stop seeing the partition, then delete the data directory + // with the table FileIO (client-side; the server never deletes data). A deletion failure leaves + // the possibly incomplete directory invisible; it must not be registered again automatically. + if (partitions.isEmpty) { + return true + } + + val onlyValueInPath = + CoreOptions.fromMap(formatTable.options()).formatTablePartitionOnlyValueInPath() + // Resolve (and path-safety validate) every partition directory before any mutation, so a + // traversal attempt ('.'/'..') fails the whole DROP before unregistering anything. + val partitionPaths = partitions.map { + spec => + resolvePartitionPathWithinTable( + formatTable, + orderedSpec(formatTable, spec), + onlyValueInPath) + } + logInfo("Try to drop managed format table partitions: " + partitions.mkString(",")) + requirePartitionCatalog().dropPartitions(partitions.asJava) + val fileIO = formatTable.fileIO() + partitionPaths.foreach { + partitionPath => + val deleted = fileIO.delete(partitionPath, true) + if (!deleted && fileIO.exists(partitionPath)) { + throw new java.io.IOException( + s"FileIO reported that partition directory $partitionPath was not deleted.") + } + } + true + } + + private def validateManagedDropPartition( + formatTable: FormatTable, + partition: java.util.Map[String, String]): java.util.Map[String, String] = { + val partitionKeys = formatTable.partitionKeys().asScala + if (partitionKeys.exists(key => partition.get(key) == null)) { + throw new IllegalStateException( + s"Catalog must return a complete partition spec with keys " + + s"${partitionKeys.mkString("[", ", ", "]")} for managed Format Table " + + s"${formatTable.fullName()}, but returned $partition.") + } + + val ordered = new java.util.LinkedHashMap[String, String]() + partitionKeys.foreach(key => ordered.put(key, partition.get(key))) + ordered + } + override def truncatePartitions(idents: Array[InternalRow]): Boolean = { val partitions = toPaimonPartitions(idents).toSeq.asJava val commit = table.newBatchWriteBuilder().newCommit() @@ -216,4 +339,81 @@ trait PaimonPartitionManagement extends SupportsAtomicPartitionManagement with L throw new UnsupportedOperationException("Only FileStoreTable supports create partitions.") } } + + private[spark] def createFormatTablePartitions( + rows: Array[InternalRow], + maps: Array[JMap[String, String]], + ignoreIfExists: Boolean): Unit = { + if (maps.exists(_.keySet().asScala.exists(_.equalsIgnoreCase("location")))) { + throw new UnsupportedOperationException( + s"ADD PARTITION with LOCATION is not supported for Format Table ${table.fullName()}.") + } + val formatTable = table.asInstanceOf[FormatTable] + val onlyValueInPath = + CoreOptions.fromMap(formatTable.options()).formatTablePartitionOnlyValueInPath() + val specs = toPaimonPartitions(rows).toSeq + // Resolve (and path-safety validate) every directory before mutating anything. + val partitionPaths = specs.map { + spec => + resolvePartitionPathWithinTable( + formatTable, + orderedSpec(formatTable, spec), + onlyValueInPath) + } + requirePartitionCatalog().createPartitions(specs.asJava, ignoreIfExists) + // Create the partition directories client-side (symmetric with DROP deleting them), so an + // added partition exists on the filesystem and a subsequent scan returns an empty partition + // rather than depending on lazy directory creation, matching Hive ADD PARTITION semantics. + val fileIO = formatTable.fileIO() + partitionPaths.foreach(partitionPath => fileIO.mkdirs(partitionPath)) + } + + private def orderedSpec( + formatTable: FormatTable, + spec: java.util.Map[String, String]): java.util.LinkedHashMap[String, String] = { + val ordered = new java.util.LinkedHashMap[String, String]() + formatTable.partitionKeys().asScala.foreach { + key => if (spec.containsKey(key)) ordered.put(key, spec.get(key)) + } + ordered + } + + /** + * Build the partition directory for a spec and verify it stays strictly under the table location. + * Value-only path components are validated (including rejecting '.'/'..'), and the normalized + * path is checked against the table location so neither DROP (recursive delete) nor ADD (mkdirs) + * can escape the table directory via crafted or corrupt partition values. + */ + private def resolvePartitionPathWithinTable( + formatTable: FormatTable, + orderedSpec: java.util.LinkedHashMap[String, String], + onlyValueInPath: Boolean): Path = { + PartitionPathUtils.validatePartitionSpecForPath(orderedSpec, onlyValueInPath) + val tablePath = new Path(formatTable.location()) + val partitionPath = new Path( + tablePath, + PartitionPathUtils.generatePartitionPathUtil(orderedSpec, onlyValueInPath) + ) + val normalizedTable = tablePath.toUri.normalize().getPath + val tablePrefix = if (normalizedTable.endsWith("/")) normalizedTable else normalizedTable + "/" + val normalizedPartition = partitionPath.toUri.normalize().getPath + if (!normalizedPartition.startsWith(tablePrefix)) { + throw new IllegalArgumentException( + s"Resolved partition path $partitionPath escapes the table location $tablePath for " + + s"partition spec $orderedSpec of Format Table ${formatTable.fullName()}.") + } + partitionPath + } + + private def requirePartitionCatalog(): FormatTablePartitionCatalog = { + if (partitionCatalog == null) { + throw new UnsupportedOperationException( + s"Partition catalog is not configured for managed Format Table ${table.fullName()}.") + } + partitionCatalog + } +} + +private object PaimonPartitionManagement { + val ManagedPartitionDropPageSize: Int = 1000 } diff --git a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/execution/PaimonFormatTablePartitionDdlExec.scala b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/execution/PaimonFormatTablePartitionDdlExec.scala new file mode 100644 index 000000000000..87086b30be86 --- /dev/null +++ b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/execution/PaimonFormatTablePartitionDdlExec.scala @@ -0,0 +1,145 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.spark.execution + +import org.apache.paimon.CoreOptions +import org.apache.paimon.spark.format.PaimonFormatTable + +import org.apache.spark.sql.catalyst.InternalRow +import org.apache.spark.sql.catalyst.analysis.{NoSuchPartitionsException, ResolvedPartitionSpec} +import org.apache.spark.sql.catalyst.expressions.Attribute +import org.apache.spark.sql.execution.datasources.v2.LeafV2CommandExec + +import java.util.{Map => JMap} + +import scala.collection.JavaConverters._ + +object PaimonFormatTablePartitionDdlExec { + + def isManaged(table: PaimonFormatTable): Boolean = + CoreOptions.fromMap(table.table.options()).partitionedTableInMetastore() + + private[execution] def unsupportedUnmanaged( + operation: String, + table: PaimonFormatTable): UnsupportedOperationException = + new UnsupportedOperationException( + s"$operation PARTITION is not supported for unmanaged Format Table ${table.name()}.") +} + +/** + * Adds managed Format Table partitions without Spark's client-side existence precheck. The whole + * batch and IF NOT EXISTS flag are forwarded to the catalog so it can apply them atomically. + */ +case class PaimonAddFormatTablePartitionsExec( + table: PaimonFormatTable, + partSpecs: Seq[ResolvedPartitionSpec], + ignoreIfExists: Boolean, + refreshCache: () => Unit) + extends LeafV2CommandExec { + + override protected def run(): Seq[InternalRow] = { + if (!PaimonFormatTablePartitionDdlExec.isManaged(table)) { + throw PaimonFormatTablePartitionDdlExec.unsupportedUnmanaged("ADD", table) + } + + if (partSpecs.nonEmpty) { + val properties: Array[JMap[String, String]] = + partSpecs.map(spec => spec.location.map("location" -> _).toMap.asJava).toArray + try { + table.createFormatTablePartitions( + partSpecs.map(_.ident).toArray, + properties, + ignoreIfExists) + } finally { + refreshCache() + } + } + Seq.empty + } + + override def output: Seq[Attribute] = Seq.empty +} + +/** + * Physical command providing explicit DROP semantics for managed Format Tables (unmanaged Format + * Tables always fail with a clear error). Complete specs are resolved against the catalog + * registration first: missing specs fail with [[NoSuchPartitionsException]] unless IF EXISTS was + * given, and only registered specs are unregistered and have their directories deleted, so data + * that is merely awaiting registration is never removed. Partial specs resolve to the registered + * leaf partitions they cover. + */ +case class PaimonDropFormatTablePartitionsExec( + table: PaimonFormatTable, + partSpecs: Seq[ResolvedPartitionSpec], + ifExists: Boolean, + purge: Boolean, + refreshCache: () => Unit) + extends LeafV2CommandExec { + + override protected def run(): Seq[InternalRow] = { + if (!PaimonFormatTablePartitionDdlExec.isManaged(table)) { + throw PaimonFormatTablePartitionDdlExec.unsupportedUnmanaged("DROP", table) + } + if (purge) { + // Match Spark's v2 default for purgePartitions. Managed Format Table DROP PARTITION + // already removes the partition data, so PURGE adds nothing. + throw new UnsupportedOperationException( + s"DROP PARTITION ... PURGE is not supported for managed Format Table ${table.name()}. " + + "DROP PARTITION already removes the partition data.") + } + if (partSpecs.isEmpty) { + return Seq.empty + } + + val partitionKeyCount = table.table.partitionKeys().size() + val (completeSpecs, partialSpecs) = + partSpecs.partition(_.ident.numFields == partitionKeyCount) + val registration = table + .formatTablePartitionsRegistered( + completeSpecs.map(_.names.toArray).toArray, + completeSpecs.map(_.ident).toArray) + .toSeq + val missingSpecs = completeSpecs.zip(registration).collect { case (spec, false) => spec } + if (missingSpecs.nonEmpty && !ifExists) { + throw new NoSuchPartitionsException( + table.name(), + missingSpecs.map(_.ident), + table.partitionSchema) + } + + val specsToDrop = + completeSpecs.zip(registration).collect { case (spec, true) => spec } ++ partialSpecs + if (specsToDrop.nonEmpty) { + // Managed semantics (PaimonPartitionManagement#dropFormatTablePartitions): resolve partial + // specs, unregister the exact catalog partitions, then delete their directories with the + // table FileIO client-side. A directory-deletion failure stays unregistered so partially + // deleted data is not exposed again. + try { + table.dropFormatTablePartitions( + specsToDrop.map(_.names.toArray).toArray, + specsToDrop.map(_.ident).toArray) + } finally { + refreshCache() + } + } + Seq.empty + } + + override def output: Seq[Attribute] = Seq.empty +} diff --git a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/execution/PaimonStrategy.scala b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/execution/PaimonStrategy.scala index 74c0e6c47dbd..d45dbdb1a983 100644 --- a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/execution/PaimonStrategy.scala +++ b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/execution/PaimonStrategy.scala @@ -29,6 +29,7 @@ import org.apache.paimon.spark.catalog.{SparkBaseCatalog, SupportView} import org.apache.paimon.spark.catalyst.analysis.ResolvedPaimonView import org.apache.paimon.spark.catalyst.plans.logical.{CopyIntoLocationCommand, CopyIntoLocationSource, CopyIntoTableCommand, CreateOrReplaceTagCommand, CreatePaimonView, DeleteTagCommand, DropPaimonView, LateralVectorSearch, PaimonCallCommand, PaimonDropPartitions, PaimonTableValuedFunctions, RenameTagCommand, ResolvedIdentifier, ShowPaimonViews, ShowTagsCommand, TruncatePaimonTableWithFilter} import org.apache.paimon.spark.data.SparkInternalRow +import org.apache.paimon.spark.format.PaimonFormatTable import org.apache.paimon.spark.read.VectorSearchResultUtils import org.apache.paimon.spark.schema.PaimonMetadataColumn import org.apache.paimon.table.{InnerTable, SpecialFields, Table} @@ -42,7 +43,7 @@ import org.apache.spark.sql.SparkSession import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.catalyst.analysis.{ResolvedNamespace, ResolvedTable} import org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeSet, Expression, GenericInternalRow, JoinedRow, PredicateHelper, UnsafeProjection} -import org.apache.spark.sql.catalyst.plans.logical.{CreateTableAsSelect, DescribeRelation, LogicalPlan, ReplaceTable, ReplaceTableAsSelect, ShowCreateTable} +import org.apache.spark.sql.catalyst.plans.logical.{AddPartitions, CreateTableAsSelect, DescribeRelation, DropPartitions, LogicalPlan, ReplaceTable, ReplaceTableAsSelect, ShowCreateTable} import org.apache.spark.sql.catalyst.util.ArrayData import org.apache.spark.sql.connector.catalog.{Identifier, PaimonLookupCatalog, TableCatalog} import org.apache.spark.sql.execution.{PaimonDescribeTableExec, SparkPlan, SparkStrategy} @@ -161,6 +162,37 @@ case class PaimonStrategy(spark: SparkSession) case _ => Nil } + case AddPartitions(r @ ResolvedTable(_, _, table: PaimonFormatTable, _), parts, ifNotExists) => + PaimonAddFormatTablePartitionsExec( + table, + parts.asResolvedPartitionSpecs, + ifNotExists, + recacheTable(r)) :: Nil + + case DropPartitions( + r @ ResolvedTable(_, _, table: PaimonFormatTable, _), + parts, + ifExists, + purge) => + PaimonDropFormatTablePartitionsExec( + table, + parts.asResolvedPartitionSpecs, + ifExists, + purge, + recacheTable(r)) :: Nil + + case PaimonDropPartitions( + r @ ResolvedTable(_, _, table: PaimonFormatTable, _), + parts, + ifExists, + purge) => + PaimonDropFormatTablePartitionsExec( + table, + parts.asResolvedPartitionSpecs, + ifExists, + purge, + recacheTable(r)) :: Nil + case PaimonDropPartitions( r @ ResolvedTable(_, _, table: SparkTable, _), parts, diff --git a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/format/CatalogFormatTablePartitionCatalog.scala b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/format/CatalogFormatTablePartitionCatalog.scala new file mode 100644 index 000000000000..dffac642c0e7 --- /dev/null +++ b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/format/CatalogFormatTablePartitionCatalog.scala @@ -0,0 +1,92 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.spark.format + +import org.apache.paimon.catalog.{Catalog, Identifier} +import org.apache.paimon.partition.Partition + +import java.util.{ArrayList, List => JList, Map => JMap} + +import scala.collection.JavaConverters._ + +/** Catalog-backed partition gateway for a single Format Table. */ +case class CatalogFormatTablePartitionCatalog(catalog: Catalog, identifier: Identifier) + extends FormatTablePartitionCatalog { + + override def createPartitions( + partitions: JList[JMap[String, String]], + ignoreIfExists: Boolean): Unit = { + // The flag reaches the server unchanged: the managed ADD command skips Spark's client-side + // existence precheck, so strict-ADD atomicity (reject the whole batch when any partition + // exists) is enforced by the catalog service. + catalog.createPartitions(identifier, partitions, ignoreIfExists) + } + + override def dropPartitions(partitions: JList[JMap[String, String]]): Unit = { + // Catalog contract: dropPartitions ignores non-existent partitions and, for managed format + // tables, unregisters metadata only. Data deletion is performed by the caller afterwards. + // Unregister in bounded batches: a partial-spec DROP can expand to more leaf partitions than + // a catalog service accepts in one request (e.g. dropping a year of a minute-partitioned + // table). Dropping ignores missing partitions, so a mid-way failure leaves a state a retry + // converges from. + var start = 0 + while (start < partitions.size()) { + val end = + math.min(start + CatalogFormatTablePartitionCatalog.MUTATION_BATCH_SIZE, partitions.size()) + catalog.dropPartitions(identifier, partitions.subList(start, end)) + start = end + } + } + + override def listPartitionsByNames( + partitions: JList[JMap[String, String]]): JList[JMap[String, String]] = { + toSpecs(catalog.listPartitionsByNames(identifier, partitions)) + } + + override def listPartitions( + prefix: JMap[String, String], + pageToken: String, + pageSize: Int): FormatTablePartitionPage = { + val page = catalog.listPartitionsPaged(identifier, Int.box(pageSize), pageToken, null) + val partitions = Option(page.getElements) + .map(_.asScala) + .getOrElse(Seq.empty) + .filter(partition => matchesPrefix(partition, prefix)) + .asJava + FormatTablePartitionPage(toSpecs(partitions), page.getNextPageToken) + } + + private def matchesPrefix(partition: Partition, prefix: JMap[String, String]): Boolean = { + prefix == null || prefix.asScala.forall { + case (key, value) => value == partition.spec().get(key) + } + } + + private def toSpecs(partitions: JList[Partition]): JList[JMap[String, String]] = { + val result = new ArrayList[JMap[String, String]](partitions.size()) + partitions.asScala.foreach(partition => result.add(partition.spec())) + result + } +} + +object CatalogFormatTablePartitionCatalog { + + /** Mirror of the catalog page size; keeps a single mutation request bounded. */ + val MUTATION_BATCH_SIZE = 1000 +} diff --git a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/format/FormatTablePartitionCatalog.scala b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/format/FormatTablePartitionCatalog.scala new file mode 100644 index 000000000000..79074555939c --- /dev/null +++ b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/format/FormatTablePartitionCatalog.scala @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.spark.format + +import java.util.{List => JList, Map => JMap} + +/** Spark-side gateway to catalog-managed Format Table partition metadata. */ +trait FormatTablePartitionCatalog { + + def createPartitions(partitions: JList[JMap[String, String]], ignoreIfExists: Boolean): Unit + + /** + * Unregister partitions from the catalog (metadata only, ignores missing partitions). Data + * deletion is the caller's responsibility. + */ + def dropPartitions(partitions: JList[JMap[String, String]]): Unit + + def listPartitionsByNames(partitions: JList[JMap[String, String]]): JList[JMap[String, String]] + + def listPartitions( + prefix: JMap[String, String], + pageToken: String, + pageSize: Int): FormatTablePartitionPage +} + +case class FormatTablePartitionPage(partitions: JList[JMap[String, String]], nextPageToken: String) diff --git a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/format/PaimonFormatTable.scala b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/format/PaimonFormatTable.scala index dc7effd39f72..42482c5366d9 100644 --- a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/format/PaimonFormatTable.scala +++ b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/format/PaimonFormatTable.scala @@ -41,6 +41,16 @@ case class PaimonFormatTable(table: FormatTable) with SupportsRead with SupportsWrite { + // Driver-only gateway owned and injected by SparkCatalog. It is intentionally transient. + @transient private var driverPartitionCatalog: FormatTablePartitionCatalog = null + + def this(table: FormatTable, partitionCatalog: FormatTablePartitionCatalog) = { + this(table) + driverPartitionCatalog = partitionCatalog + } + + override def partitionCatalog: FormatTablePartitionCatalog = driverPartitionCatalog + override def capabilities(): util.Set[TableCapability] = { util.EnumSet.of(BATCH_READ, BATCH_WRITE, OVERWRITE_DYNAMIC, OVERWRITE_BY_FILTER) } diff --git a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/util/OptionUtils.scala b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/util/OptionUtils.scala index 37521ceda653..31065e53244c 100644 --- a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/util/OptionUtils.scala +++ b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/util/OptionUtils.scala @@ -19,10 +19,10 @@ package org.apache.paimon.spark.util import org.apache.paimon.CoreOptions -import org.apache.paimon.catalog.Identifier +import org.apache.paimon.catalog.{CatalogUtils, Identifier} import org.apache.paimon.options.ConfigOption import org.apache.paimon.spark.{SparkCatalogOptions, SparkConnectorOptions} -import org.apache.paimon.table.Table +import org.apache.paimon.table.{FormatTable, Table} import org.apache.spark.internal.Logging import org.apache.spark.sql.SparkSession @@ -186,10 +186,44 @@ object OptionUtils extends SQLConfHelper with Logging { if (mergedOptions.isEmpty) { table } else { + normalizeManagedFormatTableDynamicOptions(table, mergedOptions) table.copy(mergedOptions).asInstanceOf[T] } } + /** + * Whether a format table is catalog-managed is a persisted property and cannot be flipped by a + * dynamic option. A dynamic {@code metastore.partitioned-table} (typically a session-global + * {@code spark.paimon.*} config that used to be a harmless no-op) is dropped so the persisted + * value always wins, warning only when it actually disagrees, instead of failing every format + * table load in that session. + */ + private def normalizeManagedFormatTableDynamicOptions( + table: Table, + dynamicOptions: JMap[String, String]): Unit = { + table match { + case formatTable: FormatTable => + val managedKey = CoreOptions.METASTORE_PARTITIONED_TABLE.key() + val persistedManaged = new CoreOptions(formatTable.options()).partitionedTableInMetastore() + if (dynamicOptions.containsKey(managedKey)) { + val dynamicManaged = new CoreOptions(dynamicOptions).partitionedTableInMetastore() + if (dynamicManaged != persistedManaged) { + logWarning(s"Ignoring dynamic option '$managedKey=$dynamicManaged' for format table " + + s"${formatTable.fullName()}: whether a format table is catalog-managed is fixed " + + s"at creation time (persisted value: $persistedManaged). Use ALTER TABLE to change it.") + } + dynamicOptions.remove(managedKey) + } + + if (persistedManaged) { + val effectiveOptions = new JHashMap[String, String](formatTable.options()) + effectiveOptions.putAll(dynamicOptions) + CatalogUtils.validateManagedFormatTableOptions(effectiveOptions) + } + case _ => + } + } + private def getMergedOptions( catalogName: String = null, ident: Identifier = null, diff --git a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/spark/sql/catalyst/parser/extensions/RewriteSparkDDLCommands.scala b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/spark/sql/catalyst/parser/extensions/RewriteSparkDDLCommands.scala index 5227d1f0f604..23ec0631c4dc 100644 --- a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/spark/sql/catalyst/parser/extensions/RewriteSparkDDLCommands.scala +++ b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/spark/sql/catalyst/parser/extensions/RewriteSparkDDLCommands.scala @@ -18,14 +18,18 @@ package org.apache.spark.sql.catalyst.parser.extensions -import org.apache.paimon.spark.catalog.SupportView -import org.apache.paimon.spark.catalyst.plans.logical.{PaimonDropPartitions, ResolvedIdentifier} +import org.apache.paimon.spark.SparkTable +import org.apache.paimon.spark.catalyst.plans.logical.PaimonDropPartitions +import org.apache.paimon.spark.execution.PaimonFormatTablePartitionDdlExec +import org.apache.paimon.spark.format.PaimonFormatTable import org.apache.spark.sql.SparkSession -import org.apache.spark.sql.catalyst.analysis.UnresolvedIdentifier +import org.apache.spark.sql.catalyst.analysis.{EliminateSubqueryAliases, UnresolvedTable} import org.apache.spark.sql.catalyst.plans.logical.{DropPartitions, LogicalPlan} import org.apache.spark.sql.catalyst.rules.Rule -import org.apache.spark.sql.connector.catalog.{CatalogManager, LookupCatalog} +import org.apache.spark.sql.connector.catalog.{CatalogManager, LookupCatalog, TableCatalog} + +import scala.util.Try case class RewriteSparkDDLCommands(spark: SparkSession) extends Rule[LogicalPlan] @@ -37,8 +41,41 @@ case class RewriteSparkDDLCommands(spark: SparkSession) // A new member was added to CreatePaimonView since spark4.0, // unapply pattern matching is not used here to ensure compatibility across multiple spark versions. - case DropPartitions(UnresolvedPaimonRelation(aliasedTable), parts, ifExists, purge) => + // + // Both Paimon data tables and catalog-managed Format Tables route DROP PARTITION through + // PaimonDropPartitions (managed Format Tables resolve partial specs through the partition + // gateway, so they must bypass Spark's strict partition-spec resolution too). A single + // extractor loads the table once and classifies it, avoiding a second catalog.loadTable per + // analyzer iteration. Unmanaged Format Tables are not matched and keep Spark's own resolution + // plus the explicit unsupported error at execution. + case DropPartitions(UnresolvedPaimonDropTarget(aliasedTable), parts, ifExists, purge) => PaimonDropPartitions(aliasedTable, parts, ifExists, purge) + } + + private object UnresolvedPaimonDropTarget { + def unapply(plan: LogicalPlan): Option[LogicalPlan] = { + EliminateSubqueryAliases(plan) match { + case UnresolvedTable(multipartIdentifier, _, _) + if isPaimonDropTarget(multipartIdentifier) => + Some(plan) + case _ => None + } + } + } + private def isPaimonDropTarget(multipartIdentifier: Seq[String]): Boolean = { + multipartIdentifier match { + case CatalogAndIdentifier(catalog: TableCatalog, ident) => + Try(catalog.loadTable(ident)) + .map { + case _: SparkTable => true + case formatTable: PaimonFormatTable => + PaimonFormatTablePartitionDdlExec.isManaged(formatTable) && + formatTable.partitionCatalog != null + case _ => false + } + .getOrElse(false) + case _ => false + } } } diff --git a/paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/util/OptionUtilsTest.scala b/paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/util/OptionUtilsTest.scala new file mode 100644 index 000000000000..4f8506c07bfd --- /dev/null +++ b/paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/util/OptionUtilsTest.scala @@ -0,0 +1,118 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.spark.util + +import org.apache.paimon.CoreOptions.{FORMAT_TABLE_IMPLEMENTATION, METASTORE_PARTITIONED_TABLE} +import org.apache.paimon.catalog.Identifier +import org.apache.paimon.fs.local.LocalFileIO +import org.apache.paimon.table.FormatTable +import org.apache.paimon.table.FormatTable.Format +import org.apache.paimon.types.{DataTypes, RowType} + +import org.apache.spark.sql.internal.SQLConf +import org.scalatest.funsuite.AnyFunSuite + +import java.util.Collections + +import scala.collection.JavaConverters._ + +/** Tests for [[OptionUtils]]. */ +class OptionUtilsTest extends AnyFunSuite { + + test("reject engine implementation from SQL conf for managed format table") { + val exception = intercept[IllegalArgumentException] { + SQLConf.withExistingConf(engineSQLConf) { + OptionUtils.copyWithSQLConf(formatTable(managed = true)) + } + } + + assert(exception.getMessage.contains(METASTORE_PARTITIONED_TABLE.key())) + assert(exception.getMessage.contains(FORMAT_TABLE_IMPLEMENTATION.key())) + } + + test("allow engine implementation from SQL conf for unmanaged format table") { + val copied = SQLConf.withExistingConf(engineSQLConf) { + OptionUtils.copyWithSQLConf(formatTable(managed = false)) + } + + assert(copied.options().get(FORMAT_TABLE_IMPLEMENTATION.key()) == "engine") + } + + test("reject invalid managed format table option from SQL conf with option context") { + val sqlConf = new SQLConf + sqlConf.setConfString(s"spark.paimon.${METASTORE_PARTITIONED_TABLE.key()}", "yes") + + val exception = intercept[IllegalArgumentException] { + SQLConf.withExistingConf(sqlConf) { + OptionUtils.copyWithSQLConf(formatTable(managed = false)) + } + } + + assert(exception.getMessage.contains("yes")) + assert(exception.getMessage.contains(METASTORE_PARTITIONED_TABLE.key())) + } + + test("session-level metastore.partitioned-table is ignored, not failed, on a managed table") { + val sqlConf = new SQLConf + sqlConf.setConfString(s"spark.paimon.${METASTORE_PARTITIONED_TABLE.key()}", "false") + + val copied = SQLConf.withExistingConf(sqlConf) { + OptionUtils.copyWithSQLConf(formatTable(managed = true)) + } + + // The persisted managed flag wins; the session-global override is dropped with a warning + // instead of failing every managed format table load in the session. + assert(copied.options().get(METASTORE_PARTITIONED_TABLE.key()) == "true") + } + + test("session-level metastore.partitioned-table is ignored, not failed, on an unmanaged table") { + val sqlConf = new SQLConf + sqlConf.setConfString(s"spark.paimon.${METASTORE_PARTITIONED_TABLE.key()}", "true") + + val copied = SQLConf.withExistingConf(sqlConf) { + OptionUtils.copyWithSQLConf(formatTable(managed = false)) + } + + assert(copied.options().get(METASTORE_PARTITIONED_TABLE.key()) == "false") + } + + private def engineSQLConf: SQLConf = { + val sqlConf = new SQLConf + sqlConf.setConfString( + s"spark.paimon.${FORMAT_TABLE_IMPLEMENTATION.key()}", + "engine" + ) + sqlConf + } + + private def formatTable(managed: Boolean): FormatTable = { + FormatTable + .builder() + .fileIO(new LocalFileIO) + .identifier(Identifier.create("test_db", "format_table")) + .rowType(RowType.of(DataTypes.INT(), DataTypes.STRING())) + .partitionKeys(Collections.singletonList("dt")) + .location("file:///tmp/test_db.db/format_table") + .format(Format.PARQUET) + .options(Map( + METASTORE_PARTITIONED_TABLE.key() -> managed.toString, + FORMAT_TABLE_IMPLEMENTATION.key() -> "paimon").asJava) + .build() + } +} diff --git a/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/execution/FormatTablePartitionDdlPlanningTest.scala b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/execution/FormatTablePartitionDdlPlanningTest.scala new file mode 100644 index 000000000000..a98f2361f07a --- /dev/null +++ b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/execution/FormatTablePartitionDdlPlanningTest.scala @@ -0,0 +1,960 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.spark.execution + +import org.apache.paimon.catalog.{CatalogContext, Identifier} +import org.apache.paimon.fs.{FileIO, Path} +import org.apache.paimon.fs.local.LocalFileIO +import org.apache.paimon.options.Options +import org.apache.paimon.spark.PaimonSparkTestWithRestCatalogBase +import org.apache.paimon.spark.catalyst.plans.logical.PaimonDropPartitions +import org.apache.paimon.spark.format.{FormatTablePartitionCatalog, FormatTablePartitionPage, PaimonFormatTable} +import org.apache.paimon.table.FormatTable +import org.apache.paimon.types.DataTypes + +import org.apache.spark.sql.AnalysisException +import org.apache.spark.sql.catalyst.InternalRow +import org.apache.spark.sql.catalyst.analysis.{NoSuchPartitionsException, ResolvedPartitionSpec, ResolvedTable} +import org.apache.spark.sql.catalyst.expressions.GenericInternalRow +import org.apache.spark.sql.catalyst.plans.logical.{AddPartitions, DropPartitions} +import org.apache.spark.sql.connector.catalog.{Identifier => SparkIdentifier, TableCatalog} + +import java.io.IOException +import java.lang.reflect.InvocationTargetException +import java.nio.file.Files +import java.util.{Collections, List => JList, Map => JMap} +import java.util.concurrent.{Callable, Executors, TimeUnit} + +import scala.collection.JavaConverters._ + +class FormatTablePartitionDdlPlanningTest extends PaimonSparkTestWithRestCatalogBase { + + test("strategy preserves the complete ADD batch for the managed Format Table command") { + val (table, _) = formatTable(managed = true) + val resolved = ResolvedTable.create( + spark.sessionState.catalogManager.currentCatalog.asInstanceOf[TableCatalog], + SparkIdentifier.of(Array("test"), "format_table"), + table) + val parts = Seq(partition(20260715, 10), partition(20260716, 11)) + + val plans = PaimonStrategy(spark).apply(AddPartitions(resolved, parts, ifNotExists = true)) + + assert(plans.size == 1) + val add = plans.head.asInstanceOf[PaimonAddFormatTablePartitionsExec] + assert(add.partSpecs == parts) + assert(add.ignoreIfExists) + } + + test("managed ADD performs no client-side existence lookup and forwards one atomic batch") { + val (table, gateway) = formatTable(managed = true) + var refreshCalls = 0 + val parts = Seq(partition(20260715, 10), partition(20260716, 11)) + val command = PaimonAddFormatTablePartitionsExec( + table, + parts, + ignoreIfExists = true, + () => refreshCalls += 1) + + runCommand(command) + + assert(gateway.createCalls == 1) + assert(gateway.lookupCalls == 0) + assert(gateway.ignoreIfExists) + assert( + gateway.created.map(_.asScala.toMap) == Seq( + Map("dt" -> "20260715", "hh" -> "10"), + Map("dt" -> "20260716", "hh" -> "11"))) + assert(refreshCalls == 1) + } + + test("mock service owns partial repeats, all repeats, atomic failure, and concurrent ADD") { + val gateway = new AtomicGateway(Set(Map("dt" -> "20260715", "hh" -> "10"))) + val table = new PaimonFormatTable(rawFormatTable(managed = true), gateway) + val existing = partition(20260715, 10) + val added = partition(20260716, 11) + + runCommand( + PaimonAddFormatTablePartitionsExec( + table, + Seq(existing, added), + ignoreIfExists = true, + () => ())) + runCommand( + PaimonAddFormatTablePartitionsExec( + table, + Seq(existing, added), + ignoreIfExists = true, + () => ())) + + assert( + gateway.state == Set( + Map("dt" -> "20260715", "hh" -> "10"), + Map("dt" -> "20260716", "hh" -> "11"))) + assert(gateway.batches.take(2).forall(_.size == 2)) + + val ordinaryError = intercept[IllegalStateException] { + runCommand( + PaimonAddFormatTablePartitionsExec( + table, + Seq(existing, partition(20260717, 12)), + ignoreIfExists = false, + () => ())) + } + assert(ordinaryError.getMessage.contains("already exists")) + assert(!gateway.state.exists(_("dt") == "20260717")) + + val concurrent = partition(20260718, 13) + val executor = Executors.newFixedThreadPool(8) + try { + val tasks = (1 to 20).map { + _ => + executor.submit(new Callable[Unit] { + override def call(): Unit = runCommand( + PaimonAddFormatTablePartitionsExec( + table, + Seq(concurrent), + ignoreIfExists = true, + () => ())) + }) + } + tasks.foreach(_.get(30, TimeUnit.SECONDS)) + } finally { + executor.shutdownNow() + } + assert(gateway.state.count(_("dt") == "20260718") == 1) + } + + test("unmanaged ADD and DROP always fail with an explicit unsupported error") { + val (table, gateway) = formatTable(managed = false) + val part = partition(20260715, 10) + + val addError = intercept[UnsupportedOperationException] { + runCommand( + PaimonAddFormatTablePartitionsExec(table, Seq(part), ignoreIfExists = false, () => ())) + } + assert(addError.getMessage.contains("ADD PARTITION is not supported")) + assert(addError.getMessage.contains("unmanaged Format Table")) + + val dropError = intercept[UnsupportedOperationException] { + runCommand( + PaimonDropFormatTablePartitionsExec( + table, + Seq(part), + ifExists = true, + purge = false, + () => ())) + } + assert(dropError.getMessage.contains("DROP PARTITION is not supported")) + assert(dropError.getMessage.contains("unmanaged Format Table")) + assert(gateway.createCalls == 0) + assert(gateway.dropCalls == 0) + assert(gateway.lookupCalls == 0) + } + + test("SQL unmanaged Format Table partition DDL always fails with a clear error") { + val tableName = "unmanaged_format_partition_ddl" + withTable(tableName) { + sql( + s"CREATE TABLE $tableName (id INT, dt INT, hh INT) USING CSV " + + "TBLPROPERTIES ('format-table.implementation'='paimon') PARTITIONED BY (dt, hh)") + + val addError = intercept[Exception] { + sql(s"ALTER TABLE $tableName ADD PARTITION (dt=20260715, hh=10)").collect() + } + assert(causeMessages(addError).contains("ADD PARTITION is not supported")) + + val dropError = intercept[Exception] { + sql(s"ALTER TABLE $tableName DROP IF EXISTS PARTITION (dt=20260715, hh=10)").collect() + } + assert(causeMessages(dropError).contains("DROP PARTITION is not supported")) + + // Partial specs keep Spark's strict partition-spec resolution for unmanaged Format Tables. + intercept[AnalysisException] { + sql(s"ALTER TABLE $tableName DROP PARTITION (dt=20260715)").collect() + } + } + } + + test("rewrite routes managed Format Table DROP PARTITION through PaimonDropPartitions") { + val managedName = "managed_format_drop_rewrite" + val unmanagedName = "unmanaged_format_drop_rewrite" + withTable(managedName, unmanagedName) { + sql(s"""CREATE TABLE $managedName (id INT, dt STRING, hh STRING) + |USING CSV + |PARTITIONED BY (dt, hh) + |TBLPROPERTIES ( + | 'format-table.implementation' = 'paimon', + | 'metastore.partitioned-table' = 'true') + |""".stripMargin) + sql(s"""CREATE TABLE $unmanagedName (id INT, dt STRING, hh STRING) + |USING CSV + |PARTITIONED BY (dt, hh) + |TBLPROPERTIES ('format-table.implementation' = 'paimon') + |""".stripMargin) + + // RewriteSparkDDLCommands runs inside the extension parser, so parsePlan already returns + // the rewritten plan for catalog-managed Format Tables. + val managedPlan = spark.sessionState.sqlParser.parsePlan( + s"ALTER TABLE $managedName DROP IF EXISTS PARTITION (dt='20260715')") + assert(managedPlan.isInstanceOf[PaimonDropPartitions]) + val paimonDrop = managedPlan.asInstanceOf[PaimonDropPartitions] + assert(paimonDrop.ifExists) + assert(!paimonDrop.purge) + + // Unmanaged Format Tables keep Spark's own DropPartitions and strict spec resolution. + val unmanagedPlan = spark.sessionState.sqlParser.parsePlan( + s"ALTER TABLE $unmanagedName DROP IF EXISTS PARTITION (dt='20260715')") + assert(unmanagedPlan.isInstanceOf[DropPartitions]) + } + } + + test("SQL partial DROP on a managed Format Table unregisters the matching partitions") { + val tableName = "managed_format_partial_drop_sql" + withTable(tableName) { + sql(s"""CREATE TABLE $tableName (id INT, dt STRING, hh STRING) + |USING CSV + |PARTITIONED BY (dt, hh) + |TBLPROPERTIES ( + | 'format-table.implementation' = 'paimon', + | 'metastore.partitioned-table' = 'true') + |""".stripMargin) + registerPartitions( + tableName, + Map("dt" -> "20260715", "hh" -> "10"), + Map("dt" -> "20260715", "hh" -> "11"), + Map("dt" -> "20260716", "hh" -> "10")) + + // Leading partial spec expands to every registered leaf partition below it. + sql(s"ALTER TABLE $tableName DROP PARTITION (dt='20260715')") + assert(catalogPartitions(tableName) == Set("dt=20260716/hh=10")) + + // Complete specs honor IF EXISTS and fail loudly for unregistered partitions. + sql(s"ALTER TABLE $tableName DROP IF EXISTS PARTITION (dt='20990101', hh='00')") + val missingError = intercept[Exception] { + sql(s"ALTER TABLE $tableName DROP PARTITION (dt='20990101', hh='00')").collect() + } + assert( + Iterator + .iterate(missingError: Throwable)(_.getCause) + .takeWhile(_ != null) + .exists(_.isInstanceOf[NoSuchPartitionsException])) + assert(catalogPartitions(tableName) == Set("dt=20260716/hh=10")) + + // Non-leading partial specs resolve by partition name through the gateway. + sql(s"ALTER TABLE $tableName DROP PARTITION (hh='10')") + assert(catalogPartitions(tableName) == Set.empty) + } + } + + test("strategy preserves the DROP batch and managed DROP unregisters through the gateway") { + val (table, gateway) = formatTable(managed = true) + val resolved = ResolvedTable.create( + spark.sessionState.catalogManager.currentCatalog.asInstanceOf[TableCatalog], + SparkIdentifier.of(Array("test"), "format_table"), + table) + val parts = Seq(partition(20260715, 10)) + val logical = PaimonDropPartitions(resolved, parts, true, false) + + val plans = PaimonStrategy(spark).apply(logical) + assert(plans.size == 1) + val command = plans.head.asInstanceOf[PaimonDropFormatTablePartitionsExec] + assert(command.partSpecs == parts) + assert(command.ifExists) + assert(!command.purge) + + var refreshCalls = 0 + runCommand(command.copy(refreshCache = () => refreshCalls += 1)) + assert(gateway.lookupCalls == 1) + assert(gateway.dropCalls == 1) + assert(gateway.dropped.map(_.asScala.toMap) == Seq(Map("dt" -> "20260715", "hh" -> "10"))) + assert(refreshCalls == 1) + } + + test("strategy threads IF EXISTS and PURGE flags into the format-table DROP command") { + val (table, _) = formatTable(managed = true) + val resolved = ResolvedTable.create( + spark.sessionState.catalogManager.currentCatalog.asInstanceOf[TableCatalog], + SparkIdentifier.of(Array("test"), "format_table"), + table) + val parts = Seq(partition(20260715, 10)) + + val plans = + PaimonStrategy(spark).apply(DropPartitions(resolved, parts, ifExists = false, purge = true)) + + assert(plans.size == 1) + val command = plans.head.asInstanceOf[PaimonDropFormatTablePartitionsExec] + assert(command.partSpecs == parts) + assert(!command.ifExists) + assert(command.purge) + } + + test("managed DROP PARTITION PURGE is rejected before any catalog access") { + val (table, gateway) = formatTable(managed = true) + + val error = intercept[UnsupportedOperationException] { + runCommand( + PaimonDropFormatTablePartitionsExec( + table, + Seq(partition(20260715, 10)), + ifExists = false, + purge = true, + () => fail("PURGE must not refresh the cache"))) + } + + assert(error.getMessage.contains("PURGE")) + assert(gateway.lookupCalls == 0) + assert(gateway.dropCalls == 0) + } + + test("managed complete DROP without IF EXISTS fails for unregistered partitions") { + val fileIO = LocalFileIO.create() + val tablePath = new Path(Files.createTempDirectory("format-table-drop-unregistered").toUri) + val pendingDir = new Path(tablePath, "dt=20260715/hh=10") + var dropCalls = 0 + var refreshCalls = 0 + val gateway = new FormatTablePartitionCatalog { + override def createPartitions( + partitions: JList[JMap[String, String]], + ignoreIfExists: Boolean): Unit = {} + + override def dropPartitions(partitions: JList[JMap[String, String]]): Unit = dropCalls += 1 + + override def listPartitionsByNames( + partitions: JList[JMap[String, String]]): JList[JMap[String, String]] = + Collections.emptyList() + + override def listPartitions( + prefix: JMap[String, String], + pageToken: String, + pageSize: Int): FormatTablePartitionPage = + throw new AssertionError("Complete specs must resolve through list-by-names") + } + + try { + fileIO.mkdirs(pendingDir) + val table = new PaimonFormatTable(rawFormatTable(managed = true, tablePath.toString), gateway) + + intercept[NoSuchPartitionsException] { + runCommand( + PaimonDropFormatTablePartitionsExec( + table, + Seq(partition(20260715, 10)), + ifExists = false, + purge = false, + () => refreshCalls += 1)) + } + + // The unregistered directory (still awaiting registration) must survive untouched. + assert(dropCalls == 0) + assert(refreshCalls == 0) + assert(fileIO.exists(pendingDir)) + } finally { + fileIO.delete(tablePath, true) + } + } + + test("managed DROP IF EXISTS drops only registered partitions and preserves pending data") { + val fileIO = LocalFileIO.create() + val tablePath = new Path(Files.createTempDirectory("format-table-drop-if-exists").toUri) + val registeredSpec = Map("dt" -> "20260716", "hh" -> "11") + val registeredDir = new Path(tablePath, "dt=20260716/hh=11") + val pendingDir = new Path(tablePath, "dt=20260715/hh=10") + var lookedUp = Seq.empty[Map[String, String]] + var dropped = Seq.empty[Map[String, String]] + var refreshCalls = 0 + val gateway = new FormatTablePartitionCatalog { + override def createPartitions( + partitions: JList[JMap[String, String]], + ignoreIfExists: Boolean): Unit = {} + + override def dropPartitions(partitions: JList[JMap[String, String]]): Unit = { + dropped = partitions.asScala.map(_.asScala.toMap).toSeq + } + + override def listPartitionsByNames( + partitions: JList[JMap[String, String]]): JList[JMap[String, String]] = { + lookedUp = partitions.asScala.map(_.asScala.toMap).toSeq + partitions.asScala.filter(_.asScala.toMap == registeredSpec).asJava + } + + override def listPartitions( + prefix: JMap[String, String], + pageToken: String, + pageSize: Int): FormatTablePartitionPage = + throw new AssertionError("Complete specs must resolve through list-by-names") + } + + try { + fileIO.mkdirs(pendingDir) + fileIO.mkdirs(registeredDir) + val table = new PaimonFormatTable(rawFormatTable(managed = true, tablePath.toString), gateway) + + runCommand( + PaimonDropFormatTablePartitionsExec( + table, + Seq(partition(20260715, 10), partition(20260716, 11)), + ifExists = true, + purge = false, + () => refreshCalls += 1)) + + assert( + lookedUp == Seq( + Map("dt" -> "20260715", "hh" -> "10"), + Map("dt" -> "20260716", "hh" -> "11"))) + assert(dropped == Seq(registeredSpec)) + assert(fileIO.exists(pendingDir)) + assert(!fileIO.exists(registeredDir)) + assert(refreshCalls == 1) + } finally { + fileIO.delete(tablePath, true) + } + } + + test("managed failed directory deletion stays unregistered and preserves pending data") { + val tablePath = new Path(Files.createTempDirectory("format-table-drop-retry-safe").toUri) + val failedSpec = Map("dt" -> "20260715", "hh" -> "10") + val pendingSpec = Map("dt" -> "20260716", "hh" -> "11") + val failedDir = new Path(tablePath, "dt=20260715/hh=10") + val pendingDir = new Path(tablePath, "dt=20260716/hh=11") + var failFirstDelete = true + val fileIO = new LocalFileIO { + override def delete(path: Path, recursive: Boolean): Boolean = { + if (path == failedDir && failFirstDelete) { + failFirstDelete = false + throw new IOException("injected exact partition delete failure") + } + super.delete(path, recursive) + } + } + var registered = Set(failedSpec) + var compensationCreates = Seq.empty[(Seq[Map[String, String]], Boolean)] + def newGateway(): FormatTablePartitionCatalog = new FormatTablePartitionCatalog { + override def createPartitions( + partitions: JList[JMap[String, String]], + ignoreIfExists: Boolean): Unit = { + val specs = partitions.asScala.map(_.asScala.toMap).toSeq + compensationCreates :+= ((specs, ignoreIfExists)) + registered ++= specs + } + + override def dropPartitions(partitions: JList[JMap[String, String]]): Unit = { + registered --= partitions.asScala.map(_.asScala.toMap) + } + + override def listPartitionsByNames( + partitions: JList[JMap[String, String]]): JList[JMap[String, String]] = + partitions.asScala + .map(_.asScala.toMap) + .filter(registered.contains) + .map(_.asJava) + .asJava + + override def listPartitions( + prefix: JMap[String, String], + pageToken: String, + pageSize: Int): FormatTablePartitionPage = + throw new AssertionError("Exact specs must resolve through list-by-names") + } + var refreshCalls = 0 + + try { + fileIO.mkdirs(failedDir) + fileIO.mkdirs(pendingDir) + val firstTable = + new PaimonFormatTable( + rawFormatTable(managed = true, tablePath.toString, fileIO), + newGateway()) + + val firstError = intercept[IOException] { + runCommand( + PaimonDropFormatTablePartitionsExec( + firstTable, + Seq(partition(20260715, 10)), + ifExists = true, + purge = false, + () => refreshCalls += 1)) + } + assert(firstError.getMessage.contains("injected exact partition delete failure")) + // The failed partition stays unregistered and is never recreated automatically. + assert(registered.isEmpty) + assert(compensationCreates.isEmpty) + assert(fileIO.exists(failedDir)) + assert(fileIO.exists(pendingDir)) + assert(refreshCalls == 1) + } finally { + fileIO.delete(tablePath, true) + } + } + + test("managed ambiguous catalog response does not recreate registration") { + val tablePath = new Path(Files.createTempDirectory("format-table-drop-ambiguous").toUri) + val droppedSpec = Map("dt" -> "20260715", "hh" -> "10") + val droppedDir = new Path(tablePath, "dt=20260715/hh=10") + val pendingDir = new Path(tablePath, "dt=20260716/hh=11") + val lostResponse = new IOException("injected lost catalog DROP response") + var failAfterCommit = true + var registered = Set(droppedSpec) + var compensationCreates = Seq.empty[(Seq[Map[String, String]], Boolean)] + def newGateway(): FormatTablePartitionCatalog = new FormatTablePartitionCatalog { + override def createPartitions( + partitions: JList[JMap[String, String]], + ignoreIfExists: Boolean): Unit = { + val specs = partitions.asScala.map(_.asScala.toMap).toSeq + compensationCreates :+= ((specs, ignoreIfExists)) + registered ++= specs + } + + override def dropPartitions(partitions: JList[JMap[String, String]]): Unit = { + registered --= partitions.asScala.map(_.asScala.toMap) + if (failAfterCommit) { + failAfterCommit = false + throw lostResponse + } + } + + override def listPartitionsByNames( + partitions: JList[JMap[String, String]]): JList[JMap[String, String]] = + partitions.asScala + .map(_.asScala.toMap) + .filter(registered.contains) + .map(_.asJava) + .asJava + + override def listPartitions( + prefix: JMap[String, String], + pageToken: String, + pageSize: Int): FormatTablePartitionPage = + throw new AssertionError("Exact specs must resolve through list-by-names") + } + val firstFileIO = LocalFileIO.create + var refreshCalls = 0 + + try { + firstFileIO.mkdirs(droppedDir) + firstFileIO.mkdirs(pendingDir) + val firstTable = new PaimonFormatTable( + rawFormatTable(managed = true, tablePath.toString, firstFileIO), + newGateway()) + + val firstError = intercept[IOException] { + runCommand( + PaimonDropFormatTablePartitionsExec( + firstTable, + Seq(partition(20260715, 10)), + ifExists = true, + purge = false, + () => refreshCalls += 1)) + } + + assert(firstError eq lostResponse) + assert(registered.isEmpty) + assert(compensationCreates.isEmpty) + assert(firstFileIO.exists(droppedDir)) + assert(firstFileIO.exists(pendingDir)) + assert(refreshCalls == 1) + } finally { + firstFileIO.delete(tablePath, true) + } + } + + test("managed DROP resolves a non-leading partial spec by partition name") { + var listedPrefixes = Seq.empty[Map[String, String]] + var dropped = Seq.empty[Map[String, String]] + val matching = Map("dt" -> "20260715", "hh" -> "10") + val gateway = new FormatTablePartitionCatalog { + override def createPartitions( + partitions: JList[JMap[String, String]], + ignoreIfExists: Boolean): Unit = {} + + override def dropPartitions(partitions: JList[JMap[String, String]]): Unit = { + dropped = partitions.asScala.map(_.asScala.toMap).toSeq + } + + override def listPartitionsByNames( + partitions: JList[JMap[String, String]]): JList[JMap[String, String]] = + Collections.emptyList() + + override def listPartitions( + prefix: JMap[String, String], + pageToken: String, + pageSize: Int): FormatTablePartitionPage = { + listedPrefixes :+= prefix.asScala.toMap + FormatTablePartitionPage(Seq(matching.asJava).asJava, null) + } + } + val table = new PaimonFormatTable(rawFormatTable(managed = true), gateway) + val hhOnly = ResolvedPartitionSpec(Seq("hh"), new GenericInternalRow(Array[Any](10))) + + runCommand( + PaimonDropFormatTablePartitionsExec( + table, + Seq(hhOnly), + ifExists = false, + purge = false, + () => ())) + + assert(listedPrefixes == Seq(Map.empty)) + assert(dropped == Seq(matching)) + } + + test("managed multiple non-leading partial DROP specs share one catalog traversal") { + val first = Map("dt" -> "20260715", "hh" -> "10") + val second = Map("dt" -> "20260716", "hh" -> "11") + val third = Map("dt" -> "20260717", "hh" -> "10") + val unrelated = Map("dt" -> "20260718", "hh" -> "12") + var listedPrefixes = Seq.empty[Map[String, String]] + var listedTokens = Seq.empty[String] + var listedPageSizes = Seq.empty[Int] + var dropped = Seq.empty[Map[String, String]] + val gateway = new FormatTablePartitionCatalog { + override def createPartitions( + partitions: JList[JMap[String, String]], + ignoreIfExists: Boolean): Unit = {} + + override def dropPartitions(partitions: JList[JMap[String, String]]): Unit = { + dropped = partitions.asScala.map(_.asScala.toMap).toSeq + } + + override def listPartitionsByNames( + partitions: JList[JMap[String, String]]): JList[JMap[String, String]] = + Collections.emptyList() + + override def listPartitions( + prefix: JMap[String, String], + pageToken: String, + pageSize: Int): FormatTablePartitionPage = { + val requested = prefix.asScala.toMap + listedPrefixes :+= requested + listedTokens :+= pageToken + listedPageSizes :+= pageSize + pageToken match { + case null => + FormatTablePartitionPage(Seq(first, unrelated).map(_.asJava).asJava, "page-2") + case "page-2" => + FormatTablePartitionPage(Seq(second, third).map(_.asJava).asJava, null) + case other => + throw new AssertionError(s"Unexpected page token $other") + } + } + } + val table = new PaimonFormatTable(rawFormatTable(managed = true), gateway) + val hhTen = ResolvedPartitionSpec(Seq("hh"), new GenericInternalRow(Array[Any](10))) + val hhEleven = ResolvedPartitionSpec(Seq("hh"), new GenericInternalRow(Array[Any](11))) + + runCommand( + PaimonDropFormatTablePartitionsExec( + table, + Seq(hhTen, hhEleven), + ifExists = false, + purge = false, + () => ())) + + assert(listedPrefixes == Seq(Map.empty, Map.empty)) + assert(listedTokens == Seq(null, "page-2")) + assert(listedPageSizes == Seq(1000, 1000)) + assert(dropped == Seq(first, second, third)) + assert(dropped.distinct == dropped) + } + + test("managed DROP with an empty batch does not refresh cache") { + val (table, gateway) = formatTable(managed = true) + var refreshCalls = 0 + + runCommand( + PaimonDropFormatTablePartitionsExec( + table, + Seq.empty, + ifExists = false, + purge = false, + () => refreshCalls += 1)) + + assert(gateway.dropCalls == 0) + assert(refreshCalls == 0) + } + + test("managed partial DROP consumes every catalog page before unregistering") { + var pageTokens = Seq.empty[Option[String]] + var dropped = Seq.empty[Map[String, String]] + val first = Map("dt" -> "20260715", "hh" -> "10") + val second = Map("dt" -> "20260716", "hh" -> "10") + val gateway = new FormatTablePartitionCatalog { + override def createPartitions( + partitions: JList[JMap[String, String]], + ignoreIfExists: Boolean): Unit = {} + + override def dropPartitions(partitions: JList[JMap[String, String]]): Unit = { + dropped = partitions.asScala.map(_.asScala.toMap).toSeq + } + + override def listPartitionsByNames( + partitions: JList[JMap[String, String]]): JList[JMap[String, String]] = + Collections.emptyList() + + override def listPartitions( + prefix: JMap[String, String], + pageToken: String, + pageSize: Int): FormatTablePartitionPage = { + pageTokens :+= Option(pageToken) + Option(pageToken) match { + case None => FormatTablePartitionPage(Seq(first.asJava).asJava, "page-2") + case Some("page-2") => FormatTablePartitionPage(Seq(second.asJava).asJava, null) + case other => throw new AssertionError(s"Unexpected page token $other") + } + } + } + val table = new PaimonFormatTable(rawFormatTable(managed = true), gateway) + val hhOnly = ResolvedPartitionSpec(Seq("hh"), new GenericInternalRow(Array[Any](10))) + + runCommand( + PaimonDropFormatTablePartitionsExec( + table, + Seq(hhOnly), + ifExists = false, + purge = false, + () => ())) + + assert(pageTokens == Seq(None, Some("page-2"))) + assert(dropped == Seq(first, second)) + } + + test("managed partial DROP rejects an incomplete catalog result before mutation") { + var dropCalls = 0 + val gateway = new FormatTablePartitionCatalog { + override def createPartitions( + partitions: JList[JMap[String, String]], + ignoreIfExists: Boolean): Unit = {} + + override def dropPartitions(partitions: JList[JMap[String, String]]): Unit = dropCalls += 1 + + override def listPartitionsByNames( + partitions: JList[JMap[String, String]]): JList[JMap[String, String]] = + Collections.emptyList() + + override def listPartitions( + prefix: JMap[String, String], + pageToken: String, + pageSize: Int): FormatTablePartitionPage = + FormatTablePartitionPage(Seq(Map("dt" -> "20260715").asJava).asJava, null) + } + val table = new PaimonFormatTable(rawFormatTable(managed = true), gateway) + val hhOnly = ResolvedPartitionSpec(Seq("hh"), new GenericInternalRow(Array[Any](10))) + + val error = intercept[IllegalStateException] { + runCommand( + PaimonDropFormatTablePartitionsExec( + table, + Seq(hhOnly), + ifExists = false, + purge = false, + () => ())) + } + + assert(error.getMessage.contains("complete partition spec")) + assert(dropCalls == 0) + } + + test("managed partial DROP treats an empty next-page token as terminal") { + var listCalls = 0 + var dropCalls = 0 + val gateway = new FormatTablePartitionCatalog { + override def createPartitions( + partitions: JList[JMap[String, String]], + ignoreIfExists: Boolean): Unit = {} + + override def dropPartitions(partitions: JList[JMap[String, String]]): Unit = dropCalls += 1 + + override def listPartitionsByNames( + partitions: JList[JMap[String, String]]): JList[JMap[String, String]] = + Collections.emptyList() + + override def listPartitions( + prefix: JMap[String, String], + pageToken: String, + pageSize: Int): FormatTablePartitionPage = { + listCalls += 1 + if (listCalls > 1) { + throw new AssertionError("Empty page token must not be fetched") + } + FormatTablePartitionPage(Collections.emptyList(), "") + } + } + val table = new PaimonFormatTable(rawFormatTable(managed = true), gateway) + val hhOnly = ResolvedPartitionSpec(Seq("hh"), new GenericInternalRow(Array[Any](10))) + + runCommand( + PaimonDropFormatTablePartitionsExec( + table, + Seq(hhOnly), + ifExists = false, + purge = false, + () => ())) + + assert(listCalls == 1) + assert(dropCalls == 0) + } + + private def partition(dt: Int, hh: Int): ResolvedPartitionSpec = + ResolvedPartitionSpec(Seq("dt", "hh"), new GenericInternalRow(Array[Any](dt, hh))) + + private def formatTable(managed: Boolean): (PaimonFormatTable, RecordingGateway) = { + val gateway = new RecordingGateway + (new PaimonFormatTable(rawFormatTable(managed), gateway), gateway) + } + + private def rawFormatTable(managed: Boolean): FormatTable = + rawFormatTable( + managed, + new Path(Files.createTempDirectory("format-table-ddl-planning-test").toUri).toString) + + private def rawFormatTable(managed: Boolean, location: String): FormatTable = { + rawFormatTable(managed, location, LocalFileIO.create) + } + + private def rawFormatTable(managed: Boolean, location: String, fileIO: FileIO): FormatTable = { + val options = if (managed) { + Map("metastore.partitioned-table" -> "true") + } else { + Map.empty[String, String] + } + FormatTable + .builder() + .fileIO(fileIO) + .identifier(Identifier.create("test", "format_table")) + .rowType( + DataTypes.ROW( + DataTypes.FIELD(0, "id", DataTypes.INT()), + DataTypes.FIELD(1, "dt", DataTypes.INT()), + DataTypes.FIELD(2, "hh", DataTypes.INT()))) + .partitionKeys(Seq("dt", "hh").asJava) + .location(location) + .format(FormatTable.Format.CSV) + .options(options.asJava) + .catalogContext(CatalogContext.create(new Options)) + .build() + } + + private def registerPartitions(tableName: String, specs: Map[String, String]*): Unit = + paimonCatalog.createPartitions( + Identifier.create(dbName0, tableName), + specs.map(_.asJava).asJava, + true) + + private def catalogPartitions(tableName: String): Set[String] = + paimonCatalog + .listPartitions(Identifier.create(dbName0, tableName)) + .asScala + .map(p => Seq("dt", "hh").map(key => s"$key=${p.spec().get(key)}").mkString("/")) + .toSet + + private def runCommand(command: AnyRef): Unit = { + try { + command.getClass.getMethod("run").invoke(command) + } catch { + case e: InvocationTargetException => throw e.getCause + } + } + + private def causeMessages(error: Throwable): String = { + Iterator + .iterate(error)(_.getCause) + .takeWhile(_ != null) + .flatMap(e => Option(e.getMessage)) + .mkString(" | ") + } + + private class RecordingGateway extends FormatTablePartitionCatalog { + var createCalls = 0 + var lookupCalls = 0 + var created = Seq.empty[JMap[String, String]] + var ignoreIfExists = false + var dropCalls = 0 + var dropped = Seq.empty[JMap[String, String]] + + override def createPartitions( + partitions: JList[JMap[String, String]], + ignoreIfExists: Boolean): Unit = { + createCalls += 1 + created = partitions.asScala.toSeq + this.ignoreIfExists = ignoreIfExists + } + + override def dropPartitions(partitions: JList[JMap[String, String]]): Unit = { + dropCalls += 1 + dropped = partitions.asScala.toSeq + } + + // Reports every requested spec as registered. ADD tests assert lookupCalls == 0 because + // managed ADD must never perform a client-side existence lookup. + override def listPartitionsByNames( + partitions: JList[JMap[String, String]]): JList[JMap[String, String]] = { + lookupCalls += 1 + partitions + } + + override def listPartitions( + prefix: JMap[String, String], + pageToken: String, + pageSize: Int): FormatTablePartitionPage = + FormatTablePartitionPage(Collections.emptyList(), null) + } + + private class AtomicGateway(initial: Set[Map[String, String]]) + extends FormatTablePartitionCatalog { + + private var partitions = initial + var batches = Seq.empty[Seq[Map[String, String]]] + + def state: Set[Map[String, String]] = synchronized(partitions) + + override def createPartitions( + partitionsToCreate: JList[JMap[String, String]], + ignoreIfExists: Boolean): Unit = synchronized { + val batch = partitionsToCreate.asScala.map(_.asScala.toMap).toSeq + batches :+= batch + val duplicates = batch.filter(partitions.contains) + if (duplicates.nonEmpty && !ignoreIfExists) { + throw new IllegalStateException(s"Partition already exists: ${duplicates.head}") + } + partitions ++= batch + } + + override def dropPartitions(partitionsToDrop: JList[JMap[String, String]]): Unit = + synchronized { + partitions --= partitionsToDrop.asScala.map(_.asScala.toMap) + } + + override def listPartitionsByNames( + partitions: JList[JMap[String, String]]): JList[JMap[String, String]] = + throw new AssertionError("ADD must not perform a client-side existence lookup") + + override def listPartitions( + prefix: JMap[String, String], + pageToken: String, + pageSize: Int): FormatTablePartitionPage = synchronized { + val specs = new java.util.ArrayList[JMap[String, String]]() + partitions.foreach { + spec => + val ordered = new java.util.LinkedHashMap[String, String]() + spec.foreach { case (key, value) => ordered.put(key, value) } + specs.add(ordered) + } + FormatTablePartitionPage(specs, null) + } + } +} diff --git a/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/format/CatalogFormatTablePartitionCatalogTest.scala b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/format/CatalogFormatTablePartitionCatalogTest.scala new file mode 100644 index 000000000000..07887266083b --- /dev/null +++ b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/format/CatalogFormatTablePartitionCatalogTest.scala @@ -0,0 +1,200 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.spark.format + +import org.apache.paimon.PagedList +import org.apache.paimon.catalog.{Catalog, Identifier} +import org.apache.paimon.partition.Partition + +import org.apache.spark.SparkFunSuite + +import java.lang.reflect.{InvocationHandler, Method, Proxy} +import java.util.{Arrays, LinkedHashMap, List => JList, Map => JMap} + +import scala.collection.JavaConverters._ + +class CatalogFormatTablePartitionCatalogTest extends SparkFunSuite { + + private val identifier = Identifier.create("db", "format_table") + + test("ADD forwards the complete batch and ignoreIfExists to the catalog") { + var forwardedIdentifier: Identifier = null + var forwardedPartitions = Seq.empty[JMap[String, String]] + var forwardedFlags = Seq.empty[Boolean] + val catalog = proxyCatalog { + (_, method, args) => + method.getName match { + case "createPartitions" if args.length == 3 => + forwardedIdentifier = args(0).asInstanceOf[Identifier] + forwardedPartitions = args(1).asInstanceOf[JList[JMap[String, String]]].asScala.toSeq + forwardedFlags :+= args(2).asInstanceOf[java.lang.Boolean].booleanValue() + null + case _ => null + } + } + val partitions = Arrays.asList(spec("dt", "20260715"), spec("dt", "20260716")) + + // Strict-ADD atomicity is the catalog service's job (the managed ADD command skips Spark's + // client-side existence precheck), so the flag must reach the catalog unchanged. + adapter(catalog).createPartitions(partitions, ignoreIfExists = true) + adapter(catalog).createPartitions(partitions, ignoreIfExists = false) + + assert(forwardedFlags == Seq(true, false)) + assert(forwardedIdentifier == identifier) + assert(forwardedPartitions == partitions.asScala) + } + + test("DROP unregisters oversized expansions in bounded batches") { + var batchSizes = Seq.empty[Int] + var dropped = Seq.empty[JMap[String, String]] + val catalog = proxyCatalog { + (_, method, args) => + method.getName match { + case "dropPartitions" => + val batch = args(1).asInstanceOf[JList[JMap[String, String]]] + batchSizes :+= batch.size() + dropped ++= batch.asScala + null + case _ => null + } + } + val partitions = new java.util.ArrayList[JMap[String, String]]() + (0 until 2500).foreach(i => partitions.add(spec("dt", f"d$i%04d"))) + + adapter(catalog).dropPartitions(partitions) + + assert(batchSizes == Seq(1000, 1000, 500)) + assert(dropped == partitions.asScala) + } + + test("map catalog list-by-names and paged prefix results to partition specs") { + var namesRequest = Seq.empty[JMap[String, String]] + var pageToken: String = null + var pageSize = 0 + var namePattern: String = null + val catalog = proxyCatalog { + (_, method, args) => + method.getName match { + case "listPartitionsByNames" => + namesRequest = args(1).asInstanceOf[JList[JMap[String, String]]].asScala.toSeq + Arrays.asList(partition(spec("dt", "20260715"))) + case "listPartitionsPaged" => + pageSize = args(1).asInstanceOf[Integer] + pageToken = args(2).asInstanceOf[String] + namePattern = args(3).asInstanceOf[String] + new PagedList[Partition]( + Arrays.asList(partition(spec("dt", "20260715")), partition(spec("dt", "20260716"))), + "next") + case _ => null + } + } + val gateway = adapter(catalog) + val requested = Arrays.asList(spec("dt", "20260715"), spec("dt", "missing")) + + assert( + gateway.listPartitionsByNames(requested).asScala.map(_.asScala.toMap) == + Seq(Map("dt" -> "20260715"))) + assert(namesRequest == requested.asScala) + + val page = gateway.listPartitions(spec("dt", "20260715"), "from", 17) + assert(page.partitions.asScala.map(_.asScala.toMap) == Seq(Map("dt" -> "20260715"))) + assert(page.nextPageToken == "next") + assert(pageToken == "from") + assert(pageSize == 17) + assert(namePattern == null) + } + + test("partial partition specs are enforced client-side, not as server patterns") { + var serverPattern: String = "not-called" + val catalog = proxyCatalog { + (_, method, args) => + method.getName match { + case "listPartitionsPaged" => + serverPattern = args(3).asInstanceOf[String] + new PagedList[Partition]( + Arrays.asList( + partition(multiSpec("dt" -> "20260715", "hh" -> "10")), + partition(multiSpec("dt" -> "20260715", "hh" -> "11"))), + null) + case _ => null + } + } + + val page = adapter(catalog).listPartitions(spec("hh", "10"), null, 17) + + assert(serverPattern == null) + assert( + page.partitions.asScala.map(_.asScala.toMap) == + Seq(Map("dt" -> "20260715", "hh" -> "10"))) + } + + test("managed reads propagate catalog listing failures") { + val catalog = proxyCatalog { + (_, method, _) => + method.getName match { + case "listPartitionsByNames" | "listPartitionsPaged" => + throw new UnsupportedOperationException("managed listing unavailable") + case _ => null + } + } + val gateway = adapter(catalog) + + val pointError = intercept[UnsupportedOperationException] { + gateway.listPartitionsByNames(Arrays.asList(spec("dt", "20260715"))) + } + val pageError = intercept[UnsupportedOperationException] { + gateway.listPartitions(new LinkedHashMap[String, String](), null, 10) + } + + assert(pointError.getMessage.contains("managed listing unavailable")) + assert(pageError.getMessage.contains("managed listing unavailable")) + } + + private def adapter(catalog: Catalog): FormatTablePartitionCatalog = + CatalogFormatTablePartitionCatalog(catalog, identifier) + + private def proxyCatalog(handler: (Any, Method, Array[AnyRef]) => AnyRef): Catalog = { + Proxy + .newProxyInstance( + getClass.getClassLoader, + Array(classOf[Catalog]), + new InvocationHandler { + override def invoke(proxy: Any, method: Method, args: Array[AnyRef]): AnyRef = { + handler(proxy, method, Option(args).getOrElse(Array.empty[AnyRef])) + } + } + ) + .asInstanceOf[Catalog] + } + + private def spec(key: String, value: String): JMap[String, String] = { + val result = new LinkedHashMap[String, String]() + result.put(key, value) + result + } + + private def multiSpec(entries: (String, String)*): JMap[String, String] = { + val result = new LinkedHashMap[String, String]() + entries.foreach { case (key, value) => result.put(key, value) } + result + } + + private def partition(spec: JMap[String, String]): Partition = + new Partition(spec, 0L, 0L, 0L, 0L, 0, false) +} diff --git a/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/format/FormatTablePartitionManagementTest.scala b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/format/FormatTablePartitionManagementTest.scala new file mode 100644 index 000000000000..f62734a55ca0 --- /dev/null +++ b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/format/FormatTablePartitionManagementTest.scala @@ -0,0 +1,738 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.spark.format + +import org.apache.paimon.catalog.{CatalogContext, Identifier} +import org.apache.paimon.fs.{FileIO, Path} +import org.apache.paimon.fs.local.LocalFileIO +import org.apache.paimon.options.Options +import org.apache.paimon.table.FormatTable +import org.apache.paimon.types.DataTypes + +import org.apache.spark.SparkFunSuite +import org.apache.spark.sql.catalyst.InternalRow +import org.apache.spark.sql.catalyst.expressions.GenericInternalRow +import org.apache.spark.unsafe.types.UTF8String + +import java.lang.reflect.{InvocationHandler, InvocationTargetException, Method, Proxy} +import java.nio.file.Files +import java.util.{ArrayList, Collections, List => JList, Map => JMap} +import java.util.concurrent.{CountDownLatch, TimeUnit} + +import scala.collection.JavaConverters._ +import scala.collection.mutable + +class FormatTablePartitionManagementTest extends SparkFunSuite { + + test("managed ADD forwards the complete batch and ignoreIfExists to the partition catalog") { + var forwardedPartitions = Seq.empty[JMap[String, String]] + var forwardedIgnoreIfExists = false + + val gateway = new FormatTablePartitionCatalog { + override def createPartitions( + partitions: JList[JMap[String, String]], + ignoreIfExists: Boolean): Unit = { + forwardedPartitions = partitions.asScala.toSeq + forwardedIgnoreIfExists = ignoreIfExists + } + + override def dropPartitions(partitions: JList[JMap[String, String]]): Unit = {} + + override def listPartitionsByNames( + partitions: JList[JMap[String, String]]): JList[JMap[String, String]] = + Collections.emptyList() + + override def listPartitions( + prefix: JMap[String, String], + pageToken: String, + pageSize: Int): FormatTablePartitionPage = + FormatTablePartitionPage(Collections.emptyList(), null) + } + val sparkTable = new PaimonFormatTable(managedFormatTable(), gateway) + + val rows = Array[InternalRow]( + new GenericInternalRow(Array[Any](20260715, 10)), + new GenericInternalRow(Array[Any](20260716, 11))) + val properties = Array.fill[JMap[String, String]](2)(Collections.emptyMap()) + sparkTable.createFormatTablePartitions(rows, properties, ignoreIfExists = true) + + assert(forwardedIgnoreIfExists) + assert( + forwardedPartitions.map(_.asScala.toMap) == Seq( + Map("dt" -> "20260715", "hh" -> "10"), + Map("dt" -> "20260716", "hh" -> "11"))) + } + + test("managed ADD creates the partition directory") { + val fileIO = LocalFileIO.create() + val tablePath = new Path(Files.createTempDirectory("managed-format-add-mkdirs").toUri) + val table = managedFormatTable(fileIO, tablePath.toString) + val partitionDir = new Path(tablePath, "dt=20260715/hh=10") + try { + val sparkTable = new PaimonFormatTable(table, emptyGateway) + sparkTable.createFormatTablePartitions( + Array(partitionRow(20260715, 10)), + Array[JMap[String, String]](Collections.emptyMap()), + ignoreIfExists = true) + + // ADD PARTITION creates the directory client-side, so a scan sees an empty partition + // rather than failing on a missing directory (Hive ADD PARTITION semantics). + assert(fileIO.exists(partitionDir)) + } finally { + fileIO.delete(tablePath, true) + } + } + + test("managed DROP rejects a partition value that would escape the table location") { + var dropCalls = 0 + val gateway = new FormatTablePartitionCatalog { + override def createPartitions( + partitions: JList[JMap[String, String]], + ignoreIfExists: Boolean): Unit = {} + + override def dropPartitions(partitions: JList[JMap[String, String]]): Unit = dropCalls += 1 + + override def listPartitionsByNames( + partitions: JList[JMap[String, String]]): JList[JMap[String, String]] = + Collections.emptyList() + + override def listPartitions( + prefix: JMap[String, String], + pageToken: String, + pageSize: Int): FormatTablePartitionPage = + FormatTablePartitionPage(Collections.emptyList(), null) + } + + val sparkTable = + new PaimonFormatTable(managedStringFormatTable(onlyValueInPath = true), gateway) + val error = intercept[IllegalArgumentException] { + sparkTable.dropFormatTablePartitions( + Array(Array("dt")), + Array(new GenericInternalRow(Array[Any](UTF8String.fromString(".."))))) + } + + // Traversal values are rejected before any catalog mutation, so no partition is unregistered. + assert(error.getMessage.contains("..")) + assert(dropCalls == 0) + } + + test("managed ADD with LOCATION is rejected before any catalog RPC") { + var createCalls = 0 + val gateway = new FormatTablePartitionCatalog { + override def createPartitions( + partitions: JList[JMap[String, String]], + ignoreIfExists: Boolean): Unit = createCalls += 1 + + override def dropPartitions(partitions: JList[JMap[String, String]]): Unit = {} + + override def listPartitionsByNames( + partitions: JList[JMap[String, String]]): JList[JMap[String, String]] = + Collections.emptyList() + + override def listPartitions( + prefix: JMap[String, String], + pageToken: String, + pageSize: Int): FormatTablePartitionPage = + FormatTablePartitionPage(Collections.emptyList(), null) + } + + val sparkTable = new PaimonFormatTable(managedFormatTable(), gateway) + val error = intercept[UnsupportedOperationException] { + sparkTable.createFormatTablePartitions( + Array(new GenericInternalRow(Array[Any](20260715, 10))), + Array(Map("location" -> "file:/tmp/custom").asJava), + ignoreIfExists = true) + } + + assert(error.getMessage.contains("LOCATION")) + assert(createCalls == 0) + } + + test("managed DROP PARTITION unregisters first and then deletes the partition directory") { + var dropped = Seq.empty[JMap[String, String]] + var directoryExistedAtUnregister = false + val fileIO = LocalFileIO.create() + val tablePath = new Path(Files.createTempDirectory("managed-format-drop-ordering").toUri) + val table = managedFormatTable(fileIO, tablePath.toString) + val partitionDir = new Path(tablePath, "dt=20260715/hh=10") + + val gateway = new FormatTablePartitionCatalog { + override def createPartitions( + partitions: JList[JMap[String, String]], + ignoreIfExists: Boolean): Unit = {} + + override def dropPartitions(partitions: JList[JMap[String, String]]): Unit = { + // Ordering contract: the directory must still exist when the catalog unregisters. + directoryExistedAtUnregister = fileIO.exists(partitionDir) + dropped = partitions.asScala.toSeq + } + + override def listPartitionsByNames( + partitions: JList[JMap[String, String]]): JList[JMap[String, String]] = + Collections.emptyList() + + override def listPartitions( + prefix: JMap[String, String], + pageToken: String, + pageSize: Int): FormatTablePartitionPage = + FormatTablePartitionPage(Collections.emptyList(), null) + } + + try { + fileIO.mkdirs(partitionDir) + assert(fileIO.exists(partitionDir)) + + val sparkTable = new PaimonFormatTable(table, gateway) + assert( + sparkTable.dropFormatTablePartitions( + Array(Array("dt", "hh")), + Array(new GenericInternalRow(Array[Any](20260715, 10))))) + + assert(dropped.map(_.asScala.toMap) == Seq(Map("dt" -> "20260715", "hh" -> "10"))) + assert(directoryExistedAtUnregister) + assert(!fileIO.exists(partitionDir)) + } finally { + fileIO.delete(tablePath, true) + } + } + + test("managed direct partial DROP expands leading values to exact leaf partitions") { + val fileIO = LocalFileIO.create() + val tablePath = new Path(Files.createTempDirectory("managed-format-direct-partial-drop").toUri) + val dtDir = new Path(tablePath, "dt=20260715") + val firstDir = new Path(dtDir, "hh=10") + val secondDir = new Path(dtDir, "hh=11") + val first = partitionSpec(20260715, 10) + val second = partitionSpec(20260715, 11) + var listedPrefixes = Seq.empty[Map[String, String]] + var dropped = Seq.empty[Map[String, String]] + val gateway = new FormatTablePartitionCatalog { + override def createPartitions( + partitions: JList[JMap[String, String]], + ignoreIfExists: Boolean): Unit = {} + + override def dropPartitions(partitions: JList[JMap[String, String]]): Unit = { + dropped = partitions.asScala.map(_.asScala.toMap).toSeq + } + + override def listPartitionsByNames( + partitions: JList[JMap[String, String]]): JList[JMap[String, String]] = + Collections.emptyList() + + override def listPartitions( + prefix: JMap[String, String], + pageToken: String, + pageSize: Int): FormatTablePartitionPage = { + listedPrefixes :+= prefix.asScala.toMap + FormatTablePartitionPage(Seq(first.asJava, second.asJava).asJava, null) + } + } + + try { + fileIO.mkdirs(firstDir) + fileIO.mkdirs(secondDir) + val sparkTable = + new PaimonFormatTable(managedFormatTable(fileIO, tablePath.toString), gateway) + + sparkTable.dropFormatTablePartitions( + Array(Array("dt")), + Array(new GenericInternalRow(Array[Any](20260715)))) + + // The expansion always runs one unfiltered traversal; constraints apply client-side. + assert(listedPrefixes == Seq(Map.empty)) + assert(dropped == Seq(first, second)) + assert(!fileIO.exists(firstDir)) + assert(!fileIO.exists(secondDir)) + assert(fileIO.exists(dtDir)) + } finally { + fileIO.delete(tablePath, true) + } + } + + test("managed DROP fails when FileIO reports an undeleted partition directory") { + val delegate = LocalFileIO.create() + val tablePath = new Path(Files.createTempDirectory("managed-format-drop").toUri) + val fileIO = falseDeleteFileIO(delegate) + val table = managedFormatTable(fileIO, tablePath.toString) + val partitionDir = new Path(tablePath, "dt=20260715/hh=10") + delegate.mkdirs(partitionDir) + val gateway = new RecordingDropCatalog + + try { + val error = intercept[java.io.IOException] { + new PaimonFormatTable(table, gateway) + .dropFormatTablePartitions( + Array(Array("dt", "hh")), + Array(new GenericInternalRow(Array[Any](20260715, 10)))) + } + + assert(error.getMessage.contains("was not deleted")) + assert(error.getMessage.contains("dt=20260715")) + assert(error.getMessage.contains("hh=10")) + assert(gateway.dropCalls == 1) + assert(delegate.exists(partitionDir)) + } finally { + delegate.delete(tablePath, true) + } + } + + test("managed DROP treats an already missing partition directory as deleted") { + val delegate = LocalFileIO.create() + val tablePath = new Path(Files.createTempDirectory("managed-format-drop-missing").toUri) + val fileIO = falseDeleteFileIO(delegate) + val table = managedFormatTable(fileIO, tablePath.toString) + val gateway = new RecordingDropCatalog + + try { + assert( + new PaimonFormatTable(table, gateway) + .dropFormatTablePartitions( + Array(Array("dt", "hh")), + Array(new GenericInternalRow(Array[Any](20260715, 10))))) + assert(gateway.dropCalls == 1) + } finally { + delegate.delete(tablePath, true) + } + } + + test("managed DROP deduplicates overlapping full and partial specs before deleting") { + val delegate = LocalFileIO.create() + val tablePath = new Path(Files.createTempDirectory("managed-format-drop-overlap").toUri) + val partitionDir = new Path(tablePath, "dt=20260715/hh=10") + var deleteCalls = 0 + val fileIO = Proxy + .newProxyInstance( + classOf[FileIO].getClassLoader, + Array(classOf[FileIO]), + new InvocationHandler { + override def invoke(proxy: Any, method: Method, args: Array[AnyRef]): AnyRef = { + if (method.getName == "delete" && args(0) == partitionDir) { + deleteCalls += 1 + } + try { + method.invoke(delegate, Option(args).getOrElse(Array.empty[AnyRef]): _*) + } catch { + case error: InvocationTargetException => throw error.getCause + } + } + } + ) + .asInstanceOf[FileIO] + val table = managedFormatTable(fileIO, tablePath.toString) + val matching = partitionSpec(20260715, 10) + var dropped = Seq.empty[Map[String, String]] + val gateway = new FormatTablePartitionCatalog { + override def createPartitions( + partitions: JList[JMap[String, String]], + ignoreIfExists: Boolean): Unit = {} + + override def dropPartitions(partitions: JList[JMap[String, String]]): Unit = { + dropped = partitions.asScala.map(_.asScala.toMap).toSeq + } + + override def listPartitionsByNames( + partitions: JList[JMap[String, String]]): JList[JMap[String, String]] = + Collections.emptyList() + + override def listPartitions( + prefix: JMap[String, String], + pageToken: String, + pageSize: Int): FormatTablePartitionPage = + FormatTablePartitionPage(Seq(matching.asJava).asJava, null) + } + + try { + delegate.mkdirs(partitionDir) + val sparkTable = new PaimonFormatTable(table, gateway) + sparkTable.dropFormatTablePartitions( + Array(Array("dt", "hh"), Array("hh")), + Array(partitionRow(20260715, 10), new GenericInternalRow(Array[Any](10)))) + + assert(dropped == Seq(matching)) + assert(deleteCalls == 1) + assert(!delegate.exists(partitionDir)) + assert(delegate.exists(new Path(tablePath, "dt=20260715"))) + } finally { + delegate.delete(tablePath, true) + } + } + + test("managed partial DROP leaves catalog and data untouched when a later page fails") { + val fileIO = LocalFileIO.create() + val tablePath = new Path(Files.createTempDirectory("managed-format-drop-page-failure").toUri) + val partitionDir = new Path(tablePath, "dt=20260715/hh=10") + val matching = partitionSpec(20260715, 10) + var dropCalls = 0 + val gateway = new FormatTablePartitionCatalog { + override def createPartitions( + partitions: JList[JMap[String, String]], + ignoreIfExists: Boolean): Unit = {} + + override def dropPartitions(partitions: JList[JMap[String, String]]): Unit = dropCalls += 1 + + override def listPartitionsByNames( + partitions: JList[JMap[String, String]]): JList[JMap[String, String]] = + Collections.emptyList() + + override def listPartitions( + prefix: JMap[String, String], + pageToken: String, + pageSize: Int): FormatTablePartitionPage = { + Option(pageToken) match { + case None => FormatTablePartitionPage(Seq(matching.asJava).asJava, "page-2") + case Some("page-2") => throw new TestPartitionPageException + case other => throw new AssertionError(s"Unexpected page token $other") + } + } + } + + try { + fileIO.mkdirs(partitionDir) + val sparkTable = + new PaimonFormatTable(managedFormatTable(fileIO, tablePath.toString), gateway) + + intercept[TestPartitionPageException] { + sparkTable.dropFormatTablePartitions( + Array(Array("hh")), + Array(new GenericInternalRow(Array[Any](10)))) + } + + assert(dropCalls == 0) + assert(fileIO.exists(partitionDir)) + } finally { + fileIO.delete(tablePath, true) + } + } + + test("managed partial DROP keeps data untouched after an ambiguous catalog response") { + val fileIO = LocalFileIO.create() + val tablePath = new Path(Files.createTempDirectory("managed-format-drop-response-loss").toUri) + val firstDir = new Path(tablePath, "dt=20260715/hh=10") + val secondDir = new Path(tablePath, "dt=20260715/hh=11") + val first = partitionSpec(20260715, 10) + val second = partitionSpec(20260715, 11) + val registered = mutable.LinkedHashSet(first, second) + val responseFailure = new TestCatalogDropResponseException + var dropCalls = 0 + var failDropResponse = true + val gateway = new FormatTablePartitionCatalog { + override def createPartitions( + partitions: JList[JMap[String, String]], + ignoreIfExists: Boolean): Unit = + throw new AssertionError("An ambiguous DROP must not recreate catalog partitions") + + override def dropPartitions(partitions: JList[JMap[String, String]]): Unit = { + dropCalls += 1 + registered --= partitions.asScala.map(_.asScala.toMap) + if (failDropResponse) { + failDropResponse = false + throw responseFailure + } + } + + override def listPartitionsByNames( + partitions: JList[JMap[String, String]]): JList[JMap[String, String]] = + Collections.emptyList() + + override def listPartitions( + prefix: JMap[String, String], + pageToken: String, + pageSize: Int): FormatTablePartitionPage = + FormatTablePartitionPage(registered.toSeq.map(_.asJava).asJava, null) + } + + try { + fileIO.mkdirs(firstDir) + fileIO.mkdirs(secondDir) + val sparkTable = + new PaimonFormatTable(managedFormatTable(fileIO, tablePath.toString), gateway) + + val error = intercept[TestCatalogDropResponseException] { + sparkTable.dropFormatTablePartitions( + Array(Array("dt")), + Array(new GenericInternalRow(Array[Any](20260715)))) + } + + assert(error eq responseFailure) + assert(dropCalls == 1) + assert(registered.isEmpty) + assert(fileIO.exists(firstDir)) + assert(fileIO.exists(secondDir)) + } finally { + fileIO.delete(tablePath, true) + } + } + + test("managed IF NOT EXISTS delegates all and partial duplicate batches to the catalog") { + val first = partitionSpec(20260715, 10) + val second = partitionSpec(20260716, 11) + val third = partitionSpec(20260717, 12) + val gateway = new InMemoryPartitionCatalog(Seq(first, second)) + val sparkTable = new PaimonFormatTable(managedFormatTable(), gateway) + + sparkTable.createFormatTablePartitions( + Array(partitionRow(20260715, 10), partitionRow(20260716, 11)), + Array.fill[JMap[String, String]](2)(Collections.emptyMap()), + ignoreIfExists = true) + sparkTable.createFormatTablePartitions( + Array(partitionRow(20260716, 11), partitionRow(20260717, 12)), + Array.fill[JMap[String, String]](2)(Collections.emptyMap()), + ignoreIfExists = true) + + assert(gateway.createRequests == Seq((Seq(first, second), true), (Seq(second, third), true))) + assert(gateway.partitions == Seq(first, second, third)) + assert(gateway.lookupCalls == 0) + } + + test("managed ordinary ADD propagates the catalog duplicate error") { + val existing = partitionSpec(20260715, 10) + val gateway = new InMemoryPartitionCatalog(Seq(existing)) + val sparkTable = new PaimonFormatTable(managedFormatTable(), gateway) + + val error = intercept[TestPartitionAlreadyExistsException] { + sparkTable.createFormatTablePartitions( + Array(partitionRow(20260715, 10)), + Array[JMap[String, String]](Collections.emptyMap()), + ignoreIfExists = false) + } + + assert(error.partition == existing) + assert(gateway.lookupCalls == 0) + } + + test("concurrent managed IF NOT EXISTS ADD relies on catalog atomicity") { + val first = partitionSpec(20260715, 10) + val second = partitionSpec(20260716, 11) + val third = partitionSpec(20260717, 12) + val gateway = new InMemoryPartitionCatalog + val sparkTable = new PaimonFormatTable(managedFormatTable(), gateway) + val start = new CountDownLatch(1) + val done = new CountDownLatch(2) + val failures = Collections.synchronizedList(new ArrayList[Throwable]()) + + def add(rows: Array[InternalRow]): Thread = + new Thread(new Runnable { + override def run(): Unit = { + try { + start.await() + sparkTable.createFormatTablePartitions( + rows, + Array.fill[JMap[String, String]](rows.length)(Collections.emptyMap()), + ignoreIfExists = true) + } catch { + case error: Throwable => failures.add(error) + } finally { + done.countDown() + } + } + }) + + val left = add(Array(partitionRow(20260715, 10), partitionRow(20260716, 11))) + val right = add(Array(partitionRow(20260716, 11), partitionRow(20260717, 12))) + left.start() + right.start() + start.countDown() + + assert(done.await(10, TimeUnit.SECONDS)) + assert(failures.isEmpty) + assert(gateway.partitions.toSet == Set(first, second, third)) + assert(gateway.createRequests.size == 2) + assert(gateway.createRequests.forall(_._2)) + assert(gateway.lookupCalls == 0) + } + + test("managed ADD preserves typed special partition values") { + val gateway = new InMemoryPartitionCatalog + val sparkTable = new PaimonFormatTable(managedStringFormatTable(), gateway) + val values = Seq("a/b", "a=b", "a%", "中文 value") + val rows: Seq[InternalRow] = values + .map( + value => new GenericInternalRow(Array[Any](UTF8String.fromString(value))): InternalRow) :+ + new GenericInternalRow(Array[Any](null)) + + sparkTable.createFormatTablePartitions( + rows.toArray, + Array.fill[JMap[String, String]](rows.size)(Collections.emptyMap()), + ignoreIfExists = true) + + assert( + gateway.createRequests == Seq( + (values.map(value => Map("dt" -> value)) :+ Map("dt" -> "__DEFAULT_PARTITION__"), true))) + } + + private def emptyGateway: FormatTablePartitionCatalog = + new FormatTablePartitionCatalog { + override def createPartitions( + partitions: JList[JMap[String, String]], + ignoreIfExists: Boolean): Unit = {} + + override def dropPartitions(partitions: JList[JMap[String, String]]): Unit = {} + + override def listPartitionsByNames( + partitions: JList[JMap[String, String]]): JList[JMap[String, String]] = + Collections.emptyList() + + override def listPartitions( + prefix: JMap[String, String], + pageToken: String, + pageSize: Int): FormatTablePartitionPage = + FormatTablePartitionPage(Collections.emptyList(), null) + } + + private class InMemoryPartitionCatalog(initialPartitions: Seq[Map[String, String]] = Seq.empty) + extends FormatTablePartitionCatalog { + + private val storedPartitions = mutable.LinkedHashSet(initialPartitions: _*) + private var requests = Seq.empty[(Seq[Map[String, String]], Boolean)] + var lookupCalls = 0 + + def createRequests: Seq[(Seq[Map[String, String]], Boolean)] = synchronized(requests) + + def partitions: Seq[Map[String, String]] = synchronized(storedPartitions.toSeq) + + override def createPartitions( + partitions: JList[JMap[String, String]], + ignoreIfExists: Boolean): Unit = synchronized { + val requested = partitions.asScala.map(_.asScala.toMap).toSeq + requests :+= ((requested, ignoreIfExists)) + if (!ignoreIfExists) { + requested.find(storedPartitions.contains).foreach { + duplicate => throw new TestPartitionAlreadyExistsException(duplicate) + } + } + storedPartitions ++= requested + } + + override def dropPartitions(partitions: JList[JMap[String, String]]): Unit = synchronized { + storedPartitions --= partitions.asScala.map(_.asScala.toMap) + } + + override def listPartitionsByNames( + partitions: JList[JMap[String, String]]): JList[JMap[String, String]] = synchronized { + lookupCalls += 1 + partitions.asScala + .map(_.asScala.toMap) + .filter(storedPartitions.contains) + .map(_.asJava) + .asJava + } + + override def listPartitions( + prefix: JMap[String, String], + pageToken: String, + pageSize: Int): FormatTablePartitionPage = + throw new AssertionError("ADD must not enumerate catalog or filesystem partitions") + } + + private class TestPartitionAlreadyExistsException(val partition: Map[String, String]) + extends RuntimeException + + private class TestPartitionPageException extends RuntimeException + + private class TestCatalogDropResponseException extends RuntimeException + + private class RecordingDropCatalog extends FormatTablePartitionCatalog { + var dropCalls = 0 + + override def createPartitions( + partitions: JList[JMap[String, String]], + ignoreIfExists: Boolean): Unit = {} + + override def dropPartitions(partitions: JList[JMap[String, String]]): Unit = dropCalls += 1 + + override def listPartitionsByNames( + partitions: JList[JMap[String, String]]): JList[JMap[String, String]] = + Collections.emptyList() + + override def listPartitions( + prefix: JMap[String, String], + pageToken: String, + pageSize: Int): FormatTablePartitionPage = + FormatTablePartitionPage(Collections.emptyList(), null) + } + + private def partitionRow(dt: Int, hh: Int): InternalRow = + new GenericInternalRow(Array[Any](dt, hh)) + + private def partitionSpec(dt: Int, hh: Int): Map[String, String] = + Map("dt" -> dt.toString, "hh" -> hh.toString) + + private def uniqueTableLocation(prefix: String): String = + new Path(Files.createTempDirectory(prefix).toUri).toString + + private def managedFormatTable( + fileIO: FileIO = LocalFileIO.create, + location: String = uniqueTableLocation("format_table")): FormatTable = { + FormatTable + .builder() + .fileIO(fileIO) + .identifier(Identifier.create("test_db", "format_table")) + .rowType( + DataTypes.ROW( + DataTypes.FIELD(0, "id", DataTypes.INT()), + DataTypes.FIELD(1, "dt", DataTypes.INT()), + DataTypes.FIELD(2, "hh", DataTypes.INT()))) + .partitionKeys(Seq("dt", "hh").asJava) + .location(location) + .format(FormatTable.Format.CSV) + .options(Map("metastore.partitioned-table" -> "true").asJava) + .catalogContext(CatalogContext.create(new Options)) + .build() + } + + private def falseDeleteFileIO(delegate: FileIO): FileIO = { + Proxy + .newProxyInstance( + classOf[FileIO].getClassLoader, + Array(classOf[FileIO]), + new InvocationHandler { + override def invoke(proxy: Any, method: Method, args: Array[AnyRef]): AnyRef = { + if (method.getName == "delete" && method.getParameterCount == 2) { + java.lang.Boolean.FALSE + } else { + try { + method.invoke(delegate, Option(args).getOrElse(Array.empty[AnyRef]): _*) + } catch { + case error: InvocationTargetException => throw error.getCause + } + } + } + } + ) + .asInstanceOf[FileIO] + } + + private def managedStringFormatTable(onlyValueInPath: Boolean = false): FormatTable = { + FormatTable + .builder() + .fileIO(LocalFileIO.create) + .identifier(Identifier.create("test_db", "format_table_special")) + .rowType(DataTypes.ROW( + DataTypes.FIELD(0, "id", DataTypes.INT()), + DataTypes.FIELD(1, "dt", DataTypes.STRING()))) + .partitionKeys(Seq("dt").asJava) + .location(uniqueTableLocation("format_table_special")) + .format(FormatTable.Format.CSV) + .options(Map( + "metastore.partitioned-table" -> "true", + "format-table.partition-path-only-value" -> onlyValueInPath.toString).asJava) + .catalogContext(CatalogContext.create(new Options)) + .build() + } +} diff --git a/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/format/ManagedFormatTableCatalogTest.scala b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/format/ManagedFormatTableCatalogTest.scala new file mode 100644 index 000000000000..d699f9fa61b0 --- /dev/null +++ b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/format/ManagedFormatTableCatalogTest.scala @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.spark.format + +import org.apache.paimon.spark.{PaimonSparkTestWithRestCatalogBase, SparkCatalog} + +import org.apache.spark.sql.connector.catalog.{Identifier => SparkIdentifier} + +class ManagedFormatTableCatalogTest extends PaimonSparkTestWithRestCatalogBase { + + test("SparkCatalog injects a branch-aware partition catalog into a managed Format Table") { + val tableName = "managed_format_table_catalog_gateway" + withTable(tableName) { + sql( + s"CREATE TABLE $tableName (id INT, dt STRING) USING CSV " + + "TBLPROPERTIES ('format-table.implementation'='paimon', " + + "'metastore.partitioned-table'='true') PARTITIONED BY (dt)") + + val sparkCatalog = spark.sessionState.catalogManager + .catalog("paimon") + .asInstanceOf[SparkCatalog] + val sparkTable = sparkCatalog + .loadTable(SparkIdentifier.of(Array(dbName0), tableName)) + .asInstanceOf[PaimonFormatTable] + + assert(sparkTable.partitionCatalog != null) + } + } +} From 02eebd0d7576a9ad42d074491dff674b7262d3e2 Mon Sep 17 00:00:00 2001 From: Sun Dapeng Date: Sun, 19 Jul 2026 15:25:14 +0800 Subject: [PATCH 3/3] [spark] Add managed Format Table partition administration Add list_format_table_partitions and sync_format_table_metadata procedures for catalog-managed Format Tables. Listing pages through the catalog and embeds a filter fingerprint in page tokens so stale cursors are rejected. The sync procedure diffs the filesystem against the catalog registration, defaults to dry-run, and applies repair diffs in bounded batches. MSCK REPAIR TABLE reuses the same sync engine through an interception ahead of Spark's v2 rejection, and SHOW PARTITIONS runs through a bounded, paginated executor. --- docs/docs/spark/procedures.md | 32 + .../apache/paimon/spark/SparkProcedures.java | 6 + .../paimon/spark/procedure/BaseProcedure.java | 36 + .../ListFormatTablePartitionsProcedure.java | 193 +++ .../SyncFormatTableMetadataProcedure.java | 296 +++++ .../PaimonShowFormatTablePartitionsExec.scala | 121 ++ .../PaimonFormatTablePartitionDdlExec.scala | 31 +- .../spark/execution/PaimonStrategy.scala | 26 +- .../FormatTableMetadataProcedureTest.java | 1155 +++++++++++++++++ ...monShowFormatTablePartitionsExecTest.scala | 229 ++++ .../FormatTablePartitionDdlPlanningTest.scala | 83 +- .../ManagedFormatTableMsckRepairTest.scala | 602 +++++++++ 12 files changed, 2794 insertions(+), 16 deletions(-) create mode 100644 paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/procedure/ListFormatTablePartitionsProcedure.java create mode 100644 paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/procedure/SyncFormatTableMetadataProcedure.java create mode 100644 paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/commands/PaimonShowFormatTablePartitionsExec.scala create mode 100644 paimon-spark/paimon-spark-common/src/test/java/org/apache/paimon/spark/procedure/FormatTableMetadataProcedureTest.java create mode 100644 paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/commands/PaimonShowFormatTablePartitionsExecTest.scala create mode 100644 paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/ManagedFormatTableMsckRepairTest.scala diff --git a/docs/docs/spark/procedures.md b/docs/docs/spark/procedures.md index ed3bb3ad13f7..abbbe9f02c4f 100644 --- a/docs/docs/spark/procedures.md +++ b/docs/docs/spark/procedures.md @@ -586,5 +586,37 @@ This section introduce all available spark procedures about paimon. CALL sys.rescale(table => 'default.T', bucket_num => 16, partitions => 'dt=20250217,hh=08;dt=20250217,hh=09')
+ + list_format_table_partitions + + To list partitions of a managed Format Table (a Format Table with 'metastore.partitioned-table' = 'true') page by page. Each output row contains a partition path and the next_page_token to pass into the next call; the token is null on the last page. When a where predicate is given, each page returns the matching rows of one raw catalog page, so filtered pages can carry fewer than limit rows; keep calling with the returned token until it is null. Arguments: +

  • table: the target managed Format Table identifier. Cannot be empty.
    • +
  • where: partition predicate. Left empty for all partitions.
    • +
  • limit: the maximum number of partitions to return in one page, between 1 and 10000. Default is 1000.
    • +
  • page_token: the next_page_token returned by the previous call. Left empty for the first page.
    • + + + CALL sys.list_format_table_partitions(table => 'default.T')

    + CALL sys.list_format_table_partitions(table => 'default.T', where => 'dt >= 20250101', limit => 500)

    + CALL sys.list_format_table_partitions(table => 'default.T', page_token => 'token-from-previous-call')
    + + + + sync_format_table_metadata + + To reconcile the partition metadata of a managed Format Table (a Format Table with 'metastore.partitioned-table' = 'true') with the partition directories on the filesystem. Only partition metadata is changed, data files are never touched. Defaults to a dry run which only reports the planned actions; pass dry_run => false to apply them. 'MSCK REPAIR TABLE' is the SQL equivalent which always applies the changes. Arguments: +
  • table: the target managed Format Table identifier. Cannot be empty.
    • +
  • mode: 'ADD' registers filesystem partitions missing from the catalog, 'DROP' unregisters catalog partitions whose directories no longer exist, 'SYNC' does both. Default is 'ADD'.
    • +
  • dry_run: only report the planned actions without applying them. Default is true.
    • + + + -- report partitions that would be registered
    + CALL sys.sync_format_table_metadata(table => 'default.T')

    + -- register missing partitions
    + CALL sys.sync_format_table_metadata(table => 'default.T', dry_run => false)

    + -- fully synchronize directories and metadata
    + CALL sys.sync_format_table_metadata(table => 'default.T', mode => 'SYNC', dry_run => false)
    + + diff --git a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/SparkProcedures.java b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/SparkProcedures.java index 6a2203b9cbaa..90f2bef3ca33 100644 --- a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/SparkProcedures.java +++ b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/SparkProcedures.java @@ -39,6 +39,7 @@ import org.apache.paimon.spark.procedure.ExpireSnapshotsProcedure; import org.apache.paimon.spark.procedure.ExpireTagsProcedure; import org.apache.paimon.spark.procedure.FastForwardProcedure; +import org.apache.paimon.spark.procedure.ListFormatTablePartitionsProcedure; import org.apache.paimon.spark.procedure.MarkPartitionDoneProcedure; import org.apache.paimon.spark.procedure.MergeBranchProcedure; import org.apache.paimon.spark.procedure.MigrateDatabaseProcedure; @@ -58,6 +59,7 @@ import org.apache.paimon.spark.procedure.RollbackProcedure; import org.apache.paimon.spark.procedure.RollbackToTimestampProcedure; import org.apache.paimon.spark.procedure.RollbackToWatermarkProcedure; +import org.apache.paimon.spark.procedure.SyncFormatTableMetadataProcedure; import org.apache.paimon.spark.procedure.TriggerTagAutomaticCreationProcedure; import org.apache.paimon.shade.guava30.com.google.common.collect.ImmutableMap; @@ -127,6 +129,10 @@ private static Map> initProcedureBuilders() { "trigger_tag_automatic_creation", TriggerTagAutomaticCreationProcedure::builder); procedureBuilders.put("rewrite_file_index", RewriteFileIndexProcedure::builder); procedureBuilders.put("copy", CopyFilesProcedure::builder); + procedureBuilders.put( + "list_format_table_partitions", ListFormatTablePartitionsProcedure::builder); + procedureBuilders.put( + "sync_format_table_metadata", SyncFormatTableMetadataProcedure::builder); return procedureBuilders.build(); } } diff --git a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/procedure/BaseProcedure.java b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/procedure/BaseProcedure.java index c61f0c1fa5e9..34b8bd8ff2ad 100644 --- a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/procedure/BaseProcedure.java +++ b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/procedure/BaseProcedure.java @@ -20,6 +20,9 @@ import org.apache.paimon.spark.SparkTable; import org.apache.paimon.spark.SparkUtils; +import org.apache.paimon.spark.format.PaimonFormatTable; +import org.apache.paimon.types.RowType; +import org.apache.paimon.utils.PartitionPathUtils; import org.apache.paimon.utils.Preconditions; import org.apache.spark.sql.PaimonSparkSession$; @@ -34,6 +37,7 @@ import org.apache.spark.sql.execution.datasources.v2.DataSourceV2Relation; import org.apache.spark.sql.paimon.shims.SparkShimLoader; +import java.util.Map; import java.util.function.Function; import scala.Option; @@ -104,6 +108,38 @@ protected SparkTable loadSparkTable(Identifier ident) { } } + protected PaimonFormatTable loadFormatTable(Identifier ident) { + try { + Table table = tableCatalog.loadTable(ident); + Preconditions.checkArgument( + table instanceof PaimonFormatTable, "%s is not a Paimon Format Table", ident); + return (PaimonFormatTable) table; + } catch (NoSuchTableException e) { + throw new RuntimeException("Couldn't load Format Table " + ident, e); + } + } + + /** Validate that the table is a managed Format Table and return its partition gateway. */ + static org.apache.paimon.spark.format.FormatTablePartitionCatalog managedPartitionCatalog( + PaimonFormatTable sparkTable) { + String name = sparkTable.table().fullName(); + Preconditions.checkArgument( + new org.apache.paimon.CoreOptions(sparkTable.table().options()) + .partitionedTableInMetastore(), + "%s is not a managed Format Table", + name); + org.apache.paimon.spark.format.FormatTablePartitionCatalog partitionCatalog = + sparkTable.partitionCatalog(); + Preconditions.checkNotNull( + partitionCatalog, "Managed Format Table %s has no partition catalog gateway", name); + return partitionCatalog; + } + + protected static String partitionPath(Map spec, RowType partitionType) { + String path = PartitionPathUtils.generatePartitionPath(spec, partitionType, false); + return path.endsWith("/") ? path.substring(0, path.length() - 1) : path; + } + protected DataSourceV2Relation createRelation(Identifier ident) { return DataSourceV2Relation.create( loadSparkTable(ident), Option.apply(tableCatalog), Option.apply(ident)); diff --git a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/procedure/ListFormatTablePartitionsProcedure.java b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/procedure/ListFormatTablePartitionsProcedure.java new file mode 100644 index 000000000000..8e4b76e0dcfa --- /dev/null +++ b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/procedure/ListFormatTablePartitionsProcedure.java @@ -0,0 +1,193 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.spark.procedure; + +import org.apache.paimon.data.BinaryRow; +import org.apache.paimon.data.GenericRow; +import org.apache.paimon.data.serializer.InternalRowSerializer; +import org.apache.paimon.partition.PartitionPredicate; +import org.apache.paimon.spark.format.FormatTablePartitionCatalog; +import org.apache.paimon.spark.format.FormatTablePartitionPage; +import org.apache.paimon.spark.format.PaimonFormatTable; +import org.apache.paimon.spark.utils.SparkProcedureUtils; +import org.apache.paimon.table.FormatTable; +import org.apache.paimon.types.RowType; +import org.apache.paimon.utils.InternalRowPartitionComputer; +import org.apache.paimon.utils.Preconditions; + +import org.apache.spark.sql.catalyst.InternalRow; +import org.apache.spark.sql.connector.catalog.Identifier; +import org.apache.spark.sql.connector.catalog.TableCatalog; +import org.apache.spark.sql.types.Metadata; +import org.apache.spark.sql.types.StructField; +import org.apache.spark.sql.types.StructType; +import org.apache.spark.unsafe.types.UTF8String; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.function.Predicate; + +import static org.apache.spark.sql.types.DataTypes.IntegerType; +import static org.apache.spark.sql.types.DataTypes.StringType; + +/** Lists managed format-table partitions with bounded pagination. */ +public class ListFormatTablePartitionsProcedure extends BaseProcedure { + + private static final int DEFAULT_LIMIT = 1000; + private static final int MAX_LIMIT = 10000; + private static final ProcedureParameter[] PARAMETERS = + new ProcedureParameter[] { + ProcedureParameter.required("table", StringType), + ProcedureParameter.optional("where", StringType), + ProcedureParameter.optional("limit", IntegerType), + ProcedureParameter.optional("page_token", StringType) + }; + + private static final StructType OUTPUT_TYPE = + new StructType( + new StructField[] { + new StructField("partition", StringType, false, Metadata.empty()), + new StructField("next_page_token", StringType, true, Metadata.empty()) + }); + + protected ListFormatTablePartitionsProcedure(TableCatalog tableCatalog) { + super(tableCatalog); + } + + @Override + public ProcedureParameter[] parameters() { + return PARAMETERS; + } + + @Override + public StructType outputType() { + return OUTPUT_TYPE; + } + + @Override + public InternalRow[] call(InternalRow args) { + Identifier ident = toIdentifier(args.getString(0), PARAMETERS[0].name()); + String where = args.isNullAt(1) ? null : args.getString(1); + int limit = args.isNullAt(2) ? DEFAULT_LIMIT : args.getInt(2); + String pageToken = args.isNullAt(3) ? null : args.getString(3); + Preconditions.checkArgument( + limit > 0 && limit <= MAX_LIMIT, + "limit must be between 1 and %s, but was %s", + MAX_LIMIT, + limit); + + PaimonFormatTable sparkTable = loadFormatTable(ident); + FormatTable formatTable = sparkTable.table(); + FormatTablePartitionCatalog partitionCatalog = managedPartitionCatalog(sparkTable); + + RowType partitionType = formatTable.partitionType(); + PartitionPredicate partitionPredicate = + SparkProcedureUtils.convertToPartitionPredicate( + where, partitionType, spark(), createRelation(ident, sparkTable)); + Predicate> predicate = null; + if (partitionPredicate != null) { + InternalRowSerializer serializer = new InternalRowSerializer(partitionType); + predicate = + spec -> { + GenericRow row = + InternalRowPartitionComputer.convertSpecToInternalRow( + spec, partitionType, formatTable.defaultPartName()); + BinaryRow binaryRow = serializer.toBinaryRow(row); + return partitionPredicate.test(binaryRow); + }; + } + + FormatTablePartitionPage page = + listPage(partitionCatalog, Collections.emptyMap(), pageToken, limit, predicate); + UTF8String nextToken = + page.nextPageToken() == null ? null : UTF8String.fromString(page.nextPageToken()); + return page.partitions().stream() + .map( + spec -> + newInternalRow( + UTF8String.fromString(partitionPath(spec, partitionType)), + nextToken)) + .toArray(InternalRow[]::new); + } + + public static ProcedureBuilder builder() { + return new BaseProcedure.Builder() { + @Override + protected ListFormatTablePartitionsProcedure doBuild() { + return new ListFormatTablePartitionsProcedure(tableCatalog()); + } + }; + } + + @Override + public String description() { + return "ListFormatTablePartitionsProcedure"; + } + + /** + * Fetch one raw catalog page of {@code pageSize} partitions and, when a filter is present, + * return only the matching rows. Filtered pages may therefore be sparse (fewer than {@code + * pageSize} rows); the catalog's own next-page token is passed through verbatim. Raw pages + * whose rows are all filtered out are skipped within the same call, so an empty result always + * means the listing is exhausted. + */ + static FormatTablePartitionPage listPage( + FormatTablePartitionCatalog catalog, + Map prefix, + String pageToken, + int pageSize, + Predicate> predicate) { + String token = normalizePageToken(pageToken); + while (true) { + FormatTablePartitionPage page = catalog.listPartitions(prefix, token, pageSize); + List> partitions = page.partitions(); + // An overshooting page would silently violate the caller's limit and duplicate rows + // on the next page, so it is the one gateway contract worth asserting here. + Preconditions.checkState( + partitions.size() <= pageSize, + "Format table partition catalog returned %s partitions for a page of %s.", + partitions.size(), + pageSize); + String nextToken = normalizePageToken(page.nextPageToken()); + if (predicate != null) { + List> filtered = new ArrayList<>(); + for (Map spec : partitions) { + if (predicate.test(spec)) { + filtered.add(spec); + } + } + // The next-page token travels on result rows, so a page whose rows are all + // filtered out would lose the cursor. Skip ahead until at least one row matches + // or the listing is exhausted; an empty result therefore always means "done". + if (filtered.isEmpty() && nextToken != null) { + token = nextToken; + continue; + } + partitions = filtered; + } + return new FormatTablePartitionPage(partitions, nextToken); + } + } + + private static String normalizePageToken(String pageToken) { + return pageToken == null || pageToken.isEmpty() ? null : pageToken; + } +} diff --git a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/procedure/SyncFormatTableMetadataProcedure.java b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/procedure/SyncFormatTableMetadataProcedure.java new file mode 100644 index 000000000000..56bb498406eb --- /dev/null +++ b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/procedure/SyncFormatTableMetadataProcedure.java @@ -0,0 +1,296 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.spark.procedure; + +import org.apache.paimon.CoreOptions; +import org.apache.paimon.fs.Path; +import org.apache.paimon.spark.format.FormatTablePartitionCatalog; +import org.apache.paimon.spark.format.FormatTablePartitionPage; +import org.apache.paimon.spark.format.PaimonFormatTable; +import org.apache.paimon.table.FormatTable; +import org.apache.paimon.types.RowType; +import org.apache.paimon.utils.Pair; +import org.apache.paimon.utils.PartitionPathUtils; +import org.apache.paimon.utils.Preconditions; + +import org.apache.spark.sql.catalyst.InternalRow; +import org.apache.spark.sql.connector.catalog.Identifier; +import org.apache.spark.sql.connector.catalog.TableCatalog; +import org.apache.spark.sql.types.Metadata; +import org.apache.spark.sql.types.StructField; +import org.apache.spark.sql.types.StructType; +import org.apache.spark.unsafe.types.UTF8String; + +import java.util.ArrayList; +import java.util.Comparator; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Set; + +import static org.apache.spark.sql.types.DataTypes.BooleanType; +import static org.apache.spark.sql.types.DataTypes.StringType; + +/** Reconciles filesystem partitions with managed format-table catalog metadata. */ +public class SyncFormatTableMetadataProcedure extends BaseProcedure { + + private static final int CATALOG_PAGE_SIZE = 1000; + + private static final ProcedureParameter[] PARAMETERS = + new ProcedureParameter[] { + ProcedureParameter.required("table", StringType), + ProcedureParameter.optional("mode", StringType), + ProcedureParameter.optional("dry_run", BooleanType) + }; + + private static final StructType OUTPUT_TYPE = + new StructType( + new StructField[] { + new StructField("partition", StringType, false, Metadata.empty()), + new StructField("action", StringType, false, Metadata.empty()), + new StructField("status", StringType, false, Metadata.empty()) + }); + + protected SyncFormatTableMetadataProcedure(TableCatalog tableCatalog) { + super(tableCatalog); + } + + @Override + public ProcedureParameter[] parameters() { + return PARAMETERS; + } + + @Override + public StructType outputType() { + return OUTPUT_TYPE; + } + + @Override + public InternalRow[] call(InternalRow args) { + Identifier ident = toIdentifier(args.getString(0), PARAMETERS[0].name()); + String mode = args.isNullAt(1) ? "ADD" : args.getString(1); + boolean dryRun = args.isNullAt(2) || args.getBoolean(2); + validateMode(mode); + + PaimonFormatTable sparkTable = loadFormatTable(ident); + FormatTable formatTable = sparkTable.table(); + FormatTablePartitionCatalog partitionCatalog = managedPartitionCatalog(sparkTable); + + List, String>> actions; + try { + actions = + syncPartitions( + partitionCatalog, + listFilesystemPartitionSpecs(formatTable), + formatTable.partitionKeys(), + dryRun, + mode); + } finally { + // A failed batch response cannot tell us whether the service committed, so the + // Spark-side caches are refreshed after every non-preview attempt. + if (!dryRun) { + refreshSparkCache(ident, sparkTable); + } + } + + RowType partitionType = formatTable.partitionType(); + return actions.stream() + .map( + entry -> + newInternalRow( + UTF8String.fromString( + partitionPath(entry.getKey(), partitionType)), + UTF8String.fromString(entry.getValue()), + UTF8String.fromString( + dryRun + ? "DRY_RUN" + : ("ADD".equals(entry.getValue()) + ? "REGISTERED" + : "UNREGISTERED")))) + .toArray(InternalRow[]::new); + } + + static void validateMode(String mode) { + Preconditions.checkArgument( + "ADD".equals(mode.toUpperCase(Locale.ROOT)) + || "DROP".equals(mode.toUpperCase(Locale.ROOT)) + || "SYNC".equals(mode.toUpperCase(Locale.ROOT)), + "Unsupported mode '%s'; supported modes are ADD, DROP and SYNC", + mode); + } + + public static ProcedureBuilder builder() { + return new BaseProcedure.Builder() { + @Override + protected SyncFormatTableMetadataProcedure doBuild() { + return new SyncFormatTableMetadataProcedure(tableCatalog()); + } + }; + } + + /** + * Shared engine for {@code MSCK REPAIR TABLE [{ADD|DROP|SYNC} PARTITIONS]} on a managed Format + * Table. MSCK applies immediately (never a dry run); the flag pair follows Spark's {@code + * RepairTable} plan (plain MSCK means ADD). Returns the number of applied actions. + */ + public static int repairFormatTablePartitions( + PaimonFormatTable sparkTable, boolean addPartitions, boolean dropPartitions) { + Preconditions.checkArgument( + addPartitions || dropPartitions, + "MSCK REPAIR TABLE must enable ADD and/or DROP partitions"); + FormatTable formatTable = sparkTable.table(); + FormatTablePartitionCatalog partitionCatalog = managedPartitionCatalog(sparkTable); + String mode = addPartitions && dropPartitions ? "SYNC" : (addPartitions ? "ADD" : "DROP"); + return syncPartitions( + partitionCatalog, + listFilesystemPartitionSpecs(formatTable), + formatTable.partitionKeys(), + false, + mode) + .size(); + } + + private static List> listFilesystemPartitionSpecs(FormatTable formatTable) { + // Discover partitions from the raw directory names rather than through the table scan: + // the scan casts each value to its column type and back (e.g. month=01 -> 1), producing + // specs that can no longer round-trip to the real directory. The write path registers the + // raw directory value, so MSCK/sync must diff against the same raw values to avoid + // spuriously adding/dropping partition metadata. + boolean onlyValueInPath = + new CoreOptions(formatTable.options()).formatTablePartitionOnlyValueInPath(); + List, Path>> found = + PartitionPathUtils.searchPartSpecAndPaths( + formatTable.fileIO(), + new Path(formatTable.location()), + formatTable.partitionKeys().size(), + formatTable.partitionKeys(), + onlyValueInPath); + List> specs = new ArrayList<>(found.size()); + for (Pair, Path> pair : found) { + PartitionPathUtils.validatePartitionSpecForPath(pair.getKey(), onlyValueInPath); + specs.add(pair.getKey()); + } + return specs; + } + + /** + * Diff the filesystem partition set against the catalog registration set and, unless dryRun, + * apply the requested mode. ADD registers "directory exists but unregistered"; DROP is + * metadata-only cleanup of "registered but directory missing" (data is never touched); SYNC is + * both. Scan-completeness guard: the filesystem listing that feeds {@code filesystemPartitions} + * ({@link PartitionPathUtils#searchPartSpecAndPaths}) fails on any mid-scan LIST error instead + * of returning a truncated set, so a DROP diff can only be produced from a complete listing and + * a transient failure never deregisters partitions that still exist. + */ + static List, String>> syncPartitions( + FormatTablePartitionCatalog catalog, + List> filesystemPartitions, + List partitionKeys, + boolean dryRun, + String mode) { + String normalizedMode = mode.toUpperCase(Locale.ROOT); + Set> catalogPartitions = new HashSet<>(); + String token = null; + do { + FormatTablePartitionPage page = catalog.listPartitions(null, token, CATALOG_PAGE_SIZE); + if (page.partitions() != null) { + catalogPartitions.addAll(page.partitions()); + } + String nextToken = page.nextPageToken(); + if (nextToken != null && nextToken.isEmpty()) { + // A null or empty next-page token is terminal, matching PaimonPartitionManagement. + nextToken = null; + } + token = nextToken; + } while (token != null); + + Set> filesystemSet = new HashSet<>(filesystemPartitions); + + List> addDiff = new ArrayList<>(); + if (!"DROP".equals(normalizedMode)) { + for (Map partition : filesystemPartitions) { + if (!catalogPartitions.contains(partition)) { + addDiff.add(partition); + } + } + sortByCanonicalPath(addDiff, partitionKeys); + } + List> dropDiff = new ArrayList<>(); + if (!"ADD".equals(normalizedMode)) { + for (Map partition : catalogPartitions) { + if (!filesystemSet.contains(partition)) { + dropDiff.add(partition); + } + } + sortByCanonicalPath(dropDiff, partitionKeys); + } + + if (!dryRun) { + if (!addDiff.isEmpty()) { + applyInBatches(addDiff, batch -> catalog.createPartitions(batch, true)); + } + if (!dropDiff.isEmpty()) { + applyInBatches(dropDiff, catalog::dropPartitions); + } + } + List, String>> actions = new ArrayList<>(); + addDiff.forEach(spec -> actions.add(new java.util.AbstractMap.SimpleEntry<>(spec, "ADD"))); + dropDiff.forEach( + spec -> actions.add(new java.util.AbstractMap.SimpleEntry<>(spec, "DROP"))); + return actions; + } + + /** + * Apply a repair diff in bounded batches instead of a single unbounded catalog mutation. A + * first repair of a pre-existing table can discover far more partitions than any regular write, + * and a single request carrying all of them monopolizes a server worker and one metadata + * transaction. Registration is an idempotent upsert and unregistration ignores missing + * partitions, so a mid-way failure leaves a state a rerun converges from. + */ + private static void applyInBatches( + List> diff, + java.util.function.Consumer>> operation) { + for (int start = 0; start < diff.size(); start += CATALOG_PAGE_SIZE) { + operation.accept(diff.subList(start, Math.min(start + CATALOG_PAGE_SIZE, diff.size()))); + } + } + + /** Sort partitions by their canonical path for stable output and deterministic batches. */ + private static void sortByCanonicalPath( + List> partitions, List partitionKeys) { + partitions.sort(Comparator.comparing(partition -> canonicalPath(partition, partitionKeys))); + } + + private static String canonicalPath(Map partition, List partitionKeys) { + LinkedHashMap orderedSpec = new LinkedHashMap<>(); + for (String key : partitionKeys) { + if (partition.containsKey(key)) { + orderedSpec.put(key, partition.get(key)); + } + } + return PartitionPathUtils.generatePartitionPath(orderedSpec); + } + + @Override + public String description() { + return "SyncFormatTableMetadataProcedure"; + } +} diff --git a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/commands/PaimonShowFormatTablePartitionsExec.scala b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/commands/PaimonShowFormatTablePartitionsExec.scala new file mode 100644 index 000000000000..26ce39238896 --- /dev/null +++ b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/commands/PaimonShowFormatTablePartitionsExec.scala @@ -0,0 +1,121 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.spark.commands + +import org.apache.paimon.CoreOptions +import org.apache.paimon.spark.{SparkRow, SparkTypeUtils} +import org.apache.paimon.spark.format.PaimonFormatTable +import org.apache.paimon.spark.leafnode.PaimonLeafV2CommandExec +import org.apache.paimon.utils.{InternalRowPartitionComputer, TypeUtils} + +import org.apache.spark.sql.Row +import org.apache.spark.sql.catalyst.{CatalystTypeConverters, InternalRow} +import org.apache.spark.sql.catalyst.analysis.ResolvedPartitionSpec +import org.apache.spark.sql.catalyst.catalog.ExternalCatalogUtils.escapePathName +import org.apache.spark.sql.catalyst.expressions.Attribute +import org.apache.spark.sql.catalyst.util.CharVarcharUtils +import org.apache.spark.unsafe.types.UTF8String + +import java.util.{Collections, Map => JMap} + +import scala.collection.JavaConverters._ +import scala.collection.mutable.ArrayBuffer + +/** Physical command for bounded managed Format Table partition listing. */ +case class PaimonShowFormatTablePartitionsExec( + output: Seq[Attribute], + table: PaimonFormatTable, + partitionSpec: Option[ResolvedPartitionSpec], + maxResults: Int) + extends PaimonLeafV2CommandExec { + + override protected def run(): Seq[InternalRow] = { + if (table.partitionCatalog == null) { + throw new UnsupportedOperationException( + s"Partition catalog is not configured for managed Format Table ${table.name()}.") + } + + val prefix = toCatalogPrefix + val partitions = ArrayBuffer.empty[JMap[String, String]] + var pageToken: String = null + do { + val page = + table.partitionCatalog.listPartitions(prefix, pageToken, maxResults + 1 - partitions.size) + if (page.partitions != null) { + partitions ++= page.partitions.asScala + } + // A null or empty next-page token is terminal, matching PaimonPartitionManagement. + pageToken = Option(page.nextPageToken).filter(_.nonEmpty).orNull + } while (pageToken != null && partitions.size < maxResults + 1) + + if (partitions.size > maxResults) { + throw new IllegalStateException( + s"SHOW PARTITIONS for managed Format Table ${table.name()} exceeded the configured " + + s"result limit $maxResults. Use CALL sys.list_format_table_partitions for paginated " + + "partition listing.") + } + + val defaultPartitionName = table.table.defaultPartName() + partitions + .map(partition => formatPartition(partition, defaultPartitionName)) + .sorted + .map(value => InternalRow(UTF8String.fromString(value))) + .toSeq + } + + private def toCatalogPrefix: JMap[String, String] = { + partitionSpec match { + case None => Collections.emptyMap[String, String]() + case Some(spec) => + val partitionNames = spec.names + val rowType = TypeUtils.project(table.table.rowType(), partitionNames.asJava) + val sparkSchema = SparkTypeUtils.fromPaimonRowType(rowType) + val rowConverter = CatalystTypeConverters.createToScalaConverter( + CharVarcharUtils.replaceCharVarcharWithString(sparkSchema)) + val coreOptions = CoreOptions.fromMap(table.table.options()) + val partitionComputer = new InternalRowPartitionComputer( + coreOptions.partitionDefaultName(), + rowType, + partitionNames.toArray, + coreOptions.legacyPartitionName) + partitionComputer.generatePartValues( + new SparkRow(rowType, rowConverter(spec.ident).asInstanceOf[Row])) + } + } + + private def formatPartition( + partition: JMap[String, String], + defaultPartitionName: String): String = { + table.table + .partitionKeys() + .asScala + .map { + name => + val catalogValue = partition.get(name) + val displayValue = if (defaultPartitionName == catalogValue) "null" else catalogValue + escapePathName(name) + "=" + escapePathName(displayValue) + } + .mkString("/") + } +} + +object PaimonShowFormatTablePartitionsExec { + + val DEFAULT_MAX_RESULTS: Int = 10000 +} diff --git a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/execution/PaimonFormatTablePartitionDdlExec.scala b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/execution/PaimonFormatTablePartitionDdlExec.scala index 87086b30be86..2e7208efbd87 100644 --- a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/execution/PaimonFormatTablePartitionDdlExec.scala +++ b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/execution/PaimonFormatTablePartitionDdlExec.scala @@ -20,6 +20,7 @@ package org.apache.paimon.spark.execution import org.apache.paimon.CoreOptions import org.apache.paimon.spark.format.PaimonFormatTable +import org.apache.paimon.spark.procedure.SyncFormatTableMetadataProcedure import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.catalyst.analysis.{NoSuchPartitionsException, ResolvedPartitionSpec} @@ -81,8 +82,8 @@ case class PaimonAddFormatTablePartitionsExec( * Tables always fail with a clear error). Complete specs are resolved against the catalog * registration first: missing specs fail with [[NoSuchPartitionsException]] unless IF EXISTS was * given, and only registered specs are unregistered and have their directories deleted, so data - * that is merely awaiting registration is never removed. Partial specs resolve to the registered - * leaf partitions they cover. + * that is merely awaiting registration (e.g. by MSCK REPAIR TABLE) is never removed. Partial specs + * resolve to the registered leaf partitions they cover. */ case class PaimonDropFormatTablePartitionsExec( table: PaimonFormatTable, @@ -143,3 +144,29 @@ case class PaimonDropFormatTablePartitionsExec( override def output: Seq[Attribute] = Seq.empty } + +/** + * MSCK REPAIR TABLE for managed Format Tables, reusing the sync-procedure engine. Spark rejects + * RepairTable for v2 tables in DataSourceV2Strategy, so PaimonStrategy intercepts first; plain MSCK + * means ADD, and MSCK always applies (never a dry run). Unmanaged tables are not intercepted and + * keep Spark's own v2 rejection. + */ +case class PaimonRepairFormatTablePartitionsExec( + table: PaimonFormatTable, + addPartitions: Boolean, + dropPartitions: Boolean, + refreshCache: () => Unit) + extends LeafV2CommandExec { + + override protected def run(): Seq[InternalRow] = { + try { + SyncFormatTableMetadataProcedure + .repairFormatTablePartitions(table, addPartitions, dropPartitions) + } finally { + refreshCache() + } + Seq.empty + } + + override def output: Seq[Attribute] = Seq.empty +} diff --git a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/execution/PaimonStrategy.scala b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/execution/PaimonStrategy.scala index d45dbdb1a983..82d26c89b501 100644 --- a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/execution/PaimonStrategy.scala +++ b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/execution/PaimonStrategy.scala @@ -28,6 +28,7 @@ import org.apache.paimon.spark.{PaimonRecordReaderIterator, SparkCatalog, SparkG import org.apache.paimon.spark.catalog.{SparkBaseCatalog, SupportView} import org.apache.paimon.spark.catalyst.analysis.ResolvedPaimonView import org.apache.paimon.spark.catalyst.plans.logical.{CopyIntoLocationCommand, CopyIntoLocationSource, CopyIntoTableCommand, CreateOrReplaceTagCommand, CreatePaimonView, DeleteTagCommand, DropPaimonView, LateralVectorSearch, PaimonCallCommand, PaimonDropPartitions, PaimonTableValuedFunctions, RenameTagCommand, ResolvedIdentifier, ShowPaimonViews, ShowTagsCommand, TruncatePaimonTableWithFilter} +import org.apache.paimon.spark.commands.PaimonShowFormatTablePartitionsExec import org.apache.paimon.spark.data.SparkInternalRow import org.apache.paimon.spark.format.PaimonFormatTable import org.apache.paimon.spark.read.VectorSearchResultUtils @@ -41,9 +42,9 @@ import org.apache.spark.TaskContext import org.apache.spark.rdd.RDD import org.apache.spark.sql.SparkSession import org.apache.spark.sql.catalyst.InternalRow -import org.apache.spark.sql.catalyst.analysis.{ResolvedNamespace, ResolvedTable} +import org.apache.spark.sql.catalyst.analysis.{ResolvedNamespace, ResolvedPartitionSpec, ResolvedTable} import org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeSet, Expression, GenericInternalRow, JoinedRow, PredicateHelper, UnsafeProjection} -import org.apache.spark.sql.catalyst.plans.logical.{AddPartitions, CreateTableAsSelect, DescribeRelation, DropPartitions, LogicalPlan, ReplaceTable, ReplaceTableAsSelect, ShowCreateTable} +import org.apache.spark.sql.catalyst.plans.logical.{AddPartitions, CreateTableAsSelect, DescribeRelation, DropPartitions, LogicalPlan, RepairTable, ReplaceTable, ReplaceTableAsSelect, ShowCreateTable, ShowPartitions} import org.apache.spark.sql.catalyst.util.ArrayData import org.apache.spark.sql.connector.catalog.{Identifier, PaimonLookupCatalog, TableCatalog} import org.apache.spark.sql.execution.{PaimonDescribeTableExec, SparkPlan, SparkStrategy} @@ -169,6 +170,27 @@ case class PaimonStrategy(spark: SparkSession) ifNotExists, recacheTable(r)) :: Nil + // Spark's DataSourceV2Strategy rejects RepairTable for every v2 table; managed Format Tables + // support it through the sync engine, so intercept here (extension strategies run first). + // Unmanaged tables fall through and keep the upstream rejection. + case RepairTable( + r @ ResolvedTable(_, _, table: PaimonFormatTable, _), + enableAddPartitions, + enableDropPartitions) if PaimonFormatTablePartitionDdlExec.isManaged(table) => + PaimonRepairFormatTablePartitionsExec( + table, + enableAddPartitions, + enableDropPartitions, + recacheTable(r)) :: Nil + + case ShowPartitions(ResolvedTable(_, _, table: PaimonFormatTable, _), partitionSpec, output) + if PaimonFormatTablePartitionDdlExec.isManaged(table) => + PaimonShowFormatTablePartitionsExec( + output, + table, + partitionSpec.map(_.asInstanceOf[ResolvedPartitionSpec]), + PaimonShowFormatTablePartitionsExec.DEFAULT_MAX_RESULTS) :: Nil + case DropPartitions( r @ ResolvedTable(_, _, table: PaimonFormatTable, _), parts, diff --git a/paimon-spark/paimon-spark-common/src/test/java/org/apache/paimon/spark/procedure/FormatTableMetadataProcedureTest.java b/paimon-spark/paimon-spark-common/src/test/java/org/apache/paimon/spark/procedure/FormatTableMetadataProcedureTest.java new file mode 100644 index 000000000000..beba3da65cf0 --- /dev/null +++ b/paimon-spark/paimon-spark-common/src/test/java/org/apache/paimon/spark/procedure/FormatTableMetadataProcedureTest.java @@ -0,0 +1,1155 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.spark.procedure; + +import org.apache.paimon.CoreOptions; +import org.apache.paimon.catalog.Identifier; +import org.apache.paimon.fs.FileIO; +import org.apache.paimon.fs.FileStatus; +import org.apache.paimon.fs.Path; +import org.apache.paimon.fs.local.LocalFileIO; +import org.apache.paimon.spark.SparkProcedures; +import org.apache.paimon.spark.format.FormatTablePartitionCatalog; +import org.apache.paimon.spark.format.FormatTablePartitionPage; +import org.apache.paimon.spark.format.PaimonFormatTable; +import org.apache.paimon.table.FormatTable; +import org.apache.paimon.types.DataTypes; +import org.apache.paimon.types.RowType; + +import org.apache.spark.sql.SparkSession; +import org.apache.spark.sql.catalyst.InternalRow; +import org.apache.spark.sql.catalyst.expressions.GenericInternalRow; +import org.apache.spark.sql.connector.catalog.TableCatalog; +import org.apache.spark.unsafe.types.UTF8String; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.io.IOException; +import java.lang.reflect.Proxy; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Predicate; +import java.util.stream.Collectors; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatCode; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/** Tests for managed format-table metadata procedures. */ +class FormatTableMetadataProcedureTest { + + @TempDir java.nio.file.Path tempDir; + + @Test + void procedureImplementationsAreAvailable() { + assertThatCode( + () -> + Class.forName( + "org.apache.paimon.spark.procedure.ListFormatTablePartitionsProcedure")) + .doesNotThrowAnyException(); + assertThatCode( + () -> + Class.forName( + "org.apache.paimon.spark.procedure.SyncFormatTableMetadataProcedure")) + .doesNotThrowAnyException(); + } + + @Test + void implementationsExposeProcedureBuilders() throws Exception { + assertThat(Procedure.class).isAssignableFrom(ListFormatTablePartitionsProcedure.class); + assertThat(Procedure.class).isAssignableFrom(SyncFormatTableMetadataProcedure.class); + assertThat( + ListFormatTablePartitionsProcedure.class + .getDeclaredMethod("builder") + .getReturnType()) + .isEqualTo(ProcedureBuilder.class); + assertThat(SparkProcedures.names()) + .contains("list_format_table_partitions", "sync_format_table_metadata"); + assertThat( + SyncFormatTableMetadataProcedure.class + .getDeclaredMethod("builder") + .getReturnType()) + .isEqualTo(ProcedureBuilder.class); + } + + @Test + void listProcedureDeclaresPagedPredicateContract() { + SparkSession spark = + SparkSession.builder() + .master("local[1]") + .appName("list-format-table-partitions-procedure-test") + .config("spark.ui.enabled", "false") + .getOrCreate(); + try { + Procedure procedure = + ListFormatTablePartitionsProcedure.builder().withTableCatalog(null).build(); + + assertThat( + Arrays.stream(procedure.parameters()) + .map(ProcedureParameter::name) + .collect(Collectors.toList())) + .containsExactly("table", "where", "limit", "page_token"); + assertThat( + Arrays.stream(procedure.parameters()) + .map(ProcedureParameter::required) + .collect(Collectors.toList())) + .containsExactly(true, false, false, false); + assertThat(procedure.outputType().fieldNames()) + .containsExactly("partition", "next_page_token"); + } finally { + spark.stop(); + SparkSession.clearActiveSession(); + SparkSession.clearDefaultSession(); + } + } + + @Test + void syncProcedureDeclaresSafeRepairContract() { + SparkSession spark = + SparkSession.builder() + .master("local[1]") + .appName("sync-format-table-metadata-procedure-test") + .config("spark.ui.enabled", "false") + .getOrCreate(); + try { + Procedure procedure = + SyncFormatTableMetadataProcedure.builder().withTableCatalog(null).build(); + + assertThat( + Arrays.stream(procedure.parameters()) + .map(ProcedureParameter::name) + .collect(Collectors.toList())) + .containsExactly("table", "mode", "dry_run"); + assertThat( + Arrays.stream(procedure.parameters()) + .map(ProcedureParameter::required) + .collect(Collectors.toList())) + .containsExactly(true, false, false); + assertThat(procedure.outputType().fieldNames()) + .containsExactly("partition", "action", "status"); + } finally { + spark.stop(); + SparkSession.clearActiveSession(); + SparkSession.clearDefaultSession(); + } + } + + @Test + void syncProcedureCallDefaultsToDryRunAndApplyCreatesOneBatch() throws Exception { + java.nio.file.Path partitionDirectory = + Files.createDirectories(tempDir.resolve("dt=20260701")); + Files.write( + partitionDirectory.resolve("data.csv"), + Collections.singletonList("1"), + StandardCharsets.UTF_8); + + SparkSession spark = + SparkSession.builder() + .master("local[1]") + .appName("sync-format-table-metadata-call-test") + .config("spark.ui.enabled", "false") + .getOrCreate(); + try { + RecordingCatalog partitionCatalog = new RecordingCatalog(); + partitionCatalog.addPage( + null, new FormatTablePartitionPage(Collections.emptyList(), null)); + PaimonFormatTable sparkTable = + new PaimonFormatTable( + formatTable(tempDir.toUri().toString()), partitionCatalog); + AtomicInteger refreshCount = new AtomicInteger(); + Procedure procedure = + new SyncFormatTableMetadataProcedure(tableCatalog(sparkTable)) { + @Override + protected void refreshSparkCache( + org.apache.spark.sql.connector.catalog.Identifier ident, + org.apache.spark.sql.connector.catalog.Table table) { + refreshCount.incrementAndGet(); + } + }; + + InternalRow[] dryRunRows = + procedure.call( + new GenericInternalRow( + new Object[] {UTF8String.fromString("db.t"), null, null})); + + assertThat(dryRunRows).hasSize(1); + assertThat(dryRunRows[0].getUTF8String(0).toString()).isEqualTo("dt=20260701"); + assertThat(dryRunRows[0].getUTF8String(1).toString()).isEqualTo("ADD"); + assertThat(dryRunRows[0].getUTF8String(2).toString()).isEqualTo("DRY_RUN"); + assertThat(partitionCatalog.createdPartitions).isEmpty(); + assertThat(refreshCount).hasValue(0); + + InternalRow[] applyRows = + procedure.call( + new GenericInternalRow( + new Object[] {UTF8String.fromString("db.t"), null, false})); + + assertThat(applyRows).hasSize(1); + assertThat(applyRows[0].getUTF8String(2).toString()).isEqualTo("REGISTERED"); + assertThat(partitionCatalog.createdPartitions) + .containsExactly(Collections.singletonList(spec("dt", "20260701"))); + assertThat(partitionCatalog.createIgnoreFlags).containsExactly(true); + assertThat(refreshCount).hasValue(1); + } finally { + spark.stop(); + SparkSession.clearActiveSession(); + SparkSession.clearDefaultSession(); + } + } + + @Test + void syncProcedureAppliesLargeDiffInBoundedBatches() throws Exception { + int partitionCount = 1001; + for (int index = 0; index < partitionCount; index++) { + Files.createDirectories(tempDir.resolve(String.format("dt=%04d", index))); + } + + SparkSession spark = + SparkSession.builder() + .master("local[1]") + .appName("sync-format-table-metadata-batch-test") + .config("spark.ui.enabled", "false") + .getOrCreate(); + try { + RecordingCatalog partitionCatalog = new RecordingCatalog(); + partitionCatalog.addPage( + null, new FormatTablePartitionPage(Collections.emptyList(), null)); + PaimonFormatTable sparkTable = + new PaimonFormatTable( + formatTable(tempDir.toUri().toString()), partitionCatalog); + Procedure procedure = + new SyncFormatTableMetadataProcedure(tableCatalog(sparkTable)) { + @Override + protected void refreshSparkCache( + org.apache.spark.sql.connector.catalog.Identifier ident, + org.apache.spark.sql.connector.catalog.Table table) {} + }; + + InternalRow[] applyRows = + procedure.call( + new GenericInternalRow( + new Object[] {UTF8String.fromString("db.t"), null, false})); + + assertThat(applyRows).hasSize(partitionCount); + assertThat(partitionCatalog.createdPartitions).hasSize(2); + assertThat(partitionCatalog.createdPartitions.get(0)).hasSize(1000); + assertThat(partitionCatalog.createdPartitions.get(1)).hasSize(1); + assertThat(partitionCatalog.createIgnoreFlags).containsExactly(true, true); + assertThat(partitionCatalog.createdPartitions.stream().mapToInt(List::size).sum()) + .isEqualTo(partitionCount); + } finally { + spark.stop(); + SparkSession.clearActiveSession(); + SparkSession.clearDefaultSession(); + } + } + + @Test + void listProcedurePropagatesPartitionCatalogFailure() { + SparkSession spark = + SparkSession.builder() + .master("local[1]") + .appName("list-format-table-partitions-error-test") + .config("spark.ui.enabled", "false") + .getOrCreate(); + try { + SecurityException failure = new SecurityException("partition access denied"); + RecordingCatalog partitionCatalog = + new RecordingCatalog() { + @Override + public FormatTablePartitionPage listPartitions( + Map prefix, String pageToken, int pageSize) { + throw failure; + } + }; + Procedure procedure = + ListFormatTablePartitionsProcedure.builder() + .withTableCatalog( + tableCatalog( + new PaimonFormatTable(formatTable(), partitionCatalog))) + .build(); + + assertThatThrownBy( + () -> + procedure.call( + new GenericInternalRow( + new Object[] { + UTF8String.fromString("db.t"), null, 1, null + }))) + .isSameAs(failure); + } finally { + spark.stop(); + SparkSession.clearActiveSession(); + SparkSession.clearDefaultSession(); + } + } + + @Test + void listProcedureRejectsOutOfRangeLimit() { + SparkSession spark = + SparkSession.builder() + .master("local[1]") + .appName("list-format-table-partitions-limit-test") + .config("spark.ui.enabled", "false") + .getOrCreate(); + try { + RecordingCatalog partitionCatalog = new RecordingCatalog(); + Procedure procedure = + ListFormatTablePartitionsProcedure.builder() + .withTableCatalog( + tableCatalog( + new PaimonFormatTable(formatTable(), partitionCatalog))) + .build(); + + assertThatThrownBy( + () -> + procedure.call( + new GenericInternalRow( + new Object[] { + UTF8String.fromString("db.t"), null, 0, null + }))) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("limit must be between 1 and 10000"); + assertThatThrownBy( + () -> + procedure.call( + new GenericInternalRow( + new Object[] { + UTF8String.fromString("db.t"), + null, + 10001, + null + }))) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("limit must be between 1 and 10000"); + assertThat(partitionCatalog.requestedTokens).isEmpty(); + } finally { + spark.stop(); + SparkSession.clearActiveSession(); + SparkSession.clearDefaultSession(); + } + } + + @Test + void listProcedureAppliesSqlPartitionPredicatePerCatalogPage() { + SparkSession spark = + SparkSession.builder() + .master("local[1]") + .appName("list-format-table-partitions-call-test") + .config("spark.ui.enabled", "false") + .getOrCreate(); + try { + RecordingCatalog partitionCatalog = new RecordingCatalog(); + partitionCatalog.addPage( + null, + new FormatTablePartitionPage( + Arrays.asList(spec("dt", "20260701"), spec("dt", "20260702")), "p2")); + partitionCatalog.addPage( + "p2", + new FormatTablePartitionPage( + Arrays.asList(spec("dt", "20260703"), spec("dt", "20260704")), null)); + PaimonFormatTable sparkTable = new PaimonFormatTable(formatTable(), partitionCatalog); + TableCatalog tableCatalog = tableCatalog(sparkTable); + Procedure procedure = + ListFormatTablePartitionsProcedure.builder() + .withTableCatalog(tableCatalog) + .build(); + + // A filtered page is the matching subset of one raw catalog page: it may be sparse + // and carries the catalog's own next-page token verbatim. + InternalRow[] rows = + procedure.call( + new GenericInternalRow( + new Object[] { + UTF8String.fromString("db.t"), + UTF8String.fromString("dt >= '20260702'"), + 2, + null + })); + + assertThat(rows).hasSize(1); + assertThat(rows[0].getUTF8String(0).toString()).isEqualTo("dt=20260702"); + assertThat(rows[0].getUTF8String(1).toString()).isEqualTo("p2"); + assertThat(partitionCatalog.requestedTokens).containsExactly((String) null); + + InternalRow[] nextRows = + procedure.call( + new GenericInternalRow( + new Object[] { + UTF8String.fromString("db.t"), + UTF8String.fromString("dt >= '20260702'"), + 2, + UTF8String.fromString("p2") + })); + + assertThat(nextRows).hasSize(2); + assertThat(nextRows[0].getUTF8String(0).toString()).isEqualTo("dt=20260703"); + assertThat(nextRows[1].getUTF8String(0).toString()).isEqualTo("dt=20260704"); + assertThat(nextRows[0].isNullAt(1)).isTrue(); + assertThat(nextRows[1].isNullAt(1)).isTrue(); + assertThat(partitionCatalog.requestedTokens).containsExactly(null, "p2"); + assertThat(partitionCatalog.requestedPageSizes).containsExactly(2, 2); + + assertThatThrownBy( + () -> + procedure.call( + new GenericInternalRow( + new Object[] { + UTF8String.fromString("db.t"), + UTF8String.fromString("id > 0"), + 2, + null + }))) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("Only partition predicate is supported"); + } finally { + spark.stop(); + SparkSession.clearActiveSession(); + SparkSession.clearDefaultSession(); + } + } + + @Test + void filteredListPagesAreSparseAndIterateByRawTokens() { + RecordingCatalog catalog = new RecordingCatalog(); + catalog.addPage( + null, + new FormatTablePartitionPage( + Arrays.asList(spec("dt", "1"), spec("dt", "skip-a")), "p2")); + catalog.addPage( + "p2", + new FormatTablePartitionPage( + Arrays.asList(spec("dt", "skip-b"), spec("dt", "skip-c")), "tail")); + catalog.addPage( + "tail", + new FormatTablePartitionPage(Collections.singletonList(spec("dt", "3")), null)); + Predicate> numbered = + partition -> Character.isDigit(partition.get("dt").charAt(0)); + + List> matches = new ArrayList<>(); + List pageRowCounts = new ArrayList<>(); + String token = null; + do { + FormatTablePartitionPage page = + ListFormatTablePartitionsProcedure.listPage( + catalog, Collections.emptyMap(), token, 2, numbered); + pageRowCounts.add(page.partitions().size()); + matches.addAll(page.partitions()); + token = page.nextPageToken(); + } while (token != null); + + assertThat(matches).containsExactly(spec("dt", "1"), spec("dt", "3")); + // The next-page token travels on result rows, so a raw page whose rows are all filtered + // out is skipped inside the same call instead of surfacing an empty page that would lose + // the cursor: the second call transparently advances from 'p2' to 'tail'. + assertThat(pageRowCounts).containsExactly(1, 1); + assertThat(catalog.requestedTokens).containsExactly(null, "p2", "tail"); + assertThat(catalog.requestedPageSizes).containsExactly(2, 2, 2); + } + + @Test + void listPageRejectsOversizedCatalogPage() { + RecordingCatalog catalog = new RecordingCatalog(); + catalog.addPage( + null, + new FormatTablePartitionPage( + Arrays.asList(spec("dt", "1"), spec("dt", "2")), null)); + + assertThatThrownBy( + () -> + ListFormatTablePartitionsProcedure.listPage( + catalog, Collections.emptyMap(), null, 1, null)) + .isInstanceOf(IllegalStateException.class) + .hasMessageContaining("returned 2 partitions for a page of 1"); + } + + @Test + void listPageForwardsPrefixStartingTokenAndLimit() { + RecordingCatalog catalog = new RecordingCatalog(); + catalog.addPage( + "start", + new FormatTablePartitionPage( + Collections.singletonList(spec("dt", "20260701")), "next")); + + FormatTablePartitionPage result = + ListFormatTablePartitionsProcedure.listPage( + catalog, spec("dt", "202607"), "start", 1, null); + + assertThat(result.partitions()).containsExactly(spec("dt", "20260701")); + assertThat(result.nextPageToken()).isEqualTo("next"); + assertThat(catalog.requestedPrefixes).containsExactly(spec("dt", "202607")); + assertThat(catalog.requestedTokens).containsExactly("start"); + assertThat(catalog.requestedPageSizes).containsExactly(1); + } + + @Test + void listPageTreatsEmptyNextPageTokenAsTerminal() { + RecordingCatalog catalog = new RecordingCatalog(); + catalog.addPage( + null, + new FormatTablePartitionPage( + Collections.singletonList(spec("dt", "20260701")), "")); + + FormatTablePartitionPage result = + ListFormatTablePartitionsProcedure.listPage( + catalog, Collections.emptyMap(), null, 2, ignored -> true); + + assertThat(result.partitions()).containsExactly(spec("dt", "20260701")); + assertThat(result.nextPageToken()).isNull(); + assertThat(catalog.requestedTokens).containsExactly((String) null); + } + + @Test + void syncDryRunReturnsFilesystemMinusCatalogWithoutCreatingPartitions() { + RecordingCatalog catalog = new RecordingCatalog(); + catalog.addPage( + null, + new FormatTablePartitionPage( + Collections.singletonList(spec("dt", "20260701")), null)); + + List, String>> actions = + SyncFormatTableMetadataProcedure.syncPartitions( + catalog, + Arrays.asList(spec("dt", "20260701"), spec("dt", "20260702")), + Collections.singletonList("dt"), + true, + "ADD"); + + assertThat(actions).containsExactly(action(spec("dt", "20260702"), "ADD")); + assertThat(catalog.createdPartitions).isEmpty(); + } + + @Test + void syncApplyCreatesTheWholeDiffAsAnIdempotentBatch() { + RecordingCatalog catalog = new RecordingCatalog(); + catalog.addPage(null, new FormatTablePartitionPage(Collections.emptyList(), null)); + + List, String>> actions = + SyncFormatTableMetadataProcedure.syncPartitions( + catalog, + Arrays.asList(spec("dt", "20260701"), spec("dt", "20260702")), + Collections.singletonList("dt"), + false, + "ADD"); + + assertThat(actions) + .containsExactly( + action(spec("dt", "20260701"), "ADD"), + action(spec("dt", "20260702"), "ADD")); + assertThat(catalog.createdPartitions) + .containsExactly(Arrays.asList(spec("dt", "20260701"), spec("dt", "20260702"))); + assertThat(catalog.createIgnoreFlags).containsExactly(true); + } + + @Test + void syncApplyPropagatesCreatePermissionFailureAndRefreshesCache() throws Exception { + java.nio.file.Path partitionDirectory = + Files.createDirectories(tempDir.resolve("dt=20260701")); + Files.write( + partitionDirectory.resolve("data.csv"), + Collections.singletonList("1"), + StandardCharsets.UTF_8); + SparkSession spark = + SparkSession.builder() + .master("local[1]") + .appName("sync-format-table-metadata-permission-test") + .config("spark.ui.enabled", "false") + .getOrCreate(); + try { + SecurityException failure = new SecurityException("partition create denied"); + RecordingCatalog partitionCatalog = + new RecordingCatalog() { + @Override + public void createPartitions( + List> partitions, boolean ignoreIfExists) { + throw failure; + } + }; + partitionCatalog.addPage( + null, new FormatTablePartitionPage(Collections.emptyList(), null)); + AtomicInteger refreshCount = new AtomicInteger(); + Procedure procedure = + new SyncFormatTableMetadataProcedure( + tableCatalog( + new PaimonFormatTable( + formatTable(tempDir.toUri().toString()), + partitionCatalog))) { + @Override + protected void refreshSparkCache( + org.apache.spark.sql.connector.catalog.Identifier ident, + org.apache.spark.sql.connector.catalog.Table table) { + refreshCount.incrementAndGet(); + } + }; + + assertThatThrownBy( + () -> + procedure.call( + new GenericInternalRow( + new Object[] { + UTF8String.fromString("db.t"), null, false + }))) + .isSameAs(failure); + assertThat(refreshCount).hasValue(1); + } finally { + spark.stop(); + SparkSession.clearActiveSession(); + SparkSession.clearDefaultSession(); + } + } + + @Test + void syncApplyRefreshesAfterAddSucceedsAndDropFails() throws Exception { + java.nio.file.Path partitionDirectory = + Files.createDirectories(tempDir.resolve("dt=20260715")); + Files.write( + partitionDirectory.resolve("data.csv"), + Collections.singletonList("15"), + StandardCharsets.UTF_8); + + SparkSession spark = + SparkSession.builder() + .master("local[1]") + .appName("sync-format-table-metadata-partial-success-test") + .config("spark.ui.enabled", "false") + .getOrCreate(); + try { + IllegalStateException failure = + new IllegalStateException("injected partition drop failure"); + RecordingCatalog partitionCatalog = + new RecordingCatalog() { + @Override + public void dropPartitions(List> partitions) { + throw failure; + } + }; + partitionCatalog.addPage( + null, + new FormatTablePartitionPage( + Collections.singletonList(spec("dt", "20260714")), null)); + AtomicInteger refreshCount = new AtomicInteger(); + Procedure procedure = + new SyncFormatTableMetadataProcedure( + tableCatalog( + new PaimonFormatTable( + formatTable(tempDir.toUri().toString()), + partitionCatalog))) { + @Override + protected void refreshSparkCache( + org.apache.spark.sql.connector.catalog.Identifier ident, + org.apache.spark.sql.connector.catalog.Table table) { + refreshCount.incrementAndGet(); + } + }; + + assertThatThrownBy( + () -> + procedure.call( + new GenericInternalRow( + new Object[] { + UTF8String.fromString("db.t"), + UTF8String.fromString("SYNC"), + false + }))) + .isSameAs(failure); + assertThat(partitionCatalog.createdPartitions) + .containsExactly(Collections.singletonList(spec("dt", "20260715"))); + assertThat(refreshCount).hasValue(1); + } finally { + spark.stop(); + SparkSession.clearActiveSession(); + SparkSession.clearDefaultSession(); + } + } + + @Test + void syncProcedureCallFailsClosedWhenSecondCatalogPageFails() throws Exception { + java.nio.file.Path partitionDirectory = + Files.createDirectories(tempDir.resolve("dt=20260715")); + Files.write( + partitionDirectory.resolve("data.csv"), + Collections.singletonList("15"), + StandardCharsets.UTF_8); + + SparkSession spark = + SparkSession.builder() + .master("local[1]") + .appName("sync-format-table-metadata-second-page-failure-test") + .config("spark.ui.enabled", "false") + .getOrCreate(); + try { + IllegalStateException listFailure = + new IllegalStateException("injected second catalog page failure"); + AtomicInteger listCount = new AtomicInteger(); + RecordingCatalog partitionCatalog = + new RecordingCatalog() { + @Override + public FormatTablePartitionPage listPartitions( + Map prefix, String pageToken, int pageSize) { + listCount.incrementAndGet(); + if ("page-2".equals(pageToken)) { + throw listFailure; + } + return super.listPartitions(prefix, pageToken, pageSize); + } + }; + partitionCatalog.addPage( + null, + new FormatTablePartitionPage( + Collections.singletonList(spec("dt", "20260714")), "page-2")); + AtomicInteger refreshCount = new AtomicInteger(); + Procedure procedure = + new SyncFormatTableMetadataProcedure( + tableCatalog( + new PaimonFormatTable( + formatTable(tempDir.toUri().toString()), + partitionCatalog))) { + @Override + protected void refreshSparkCache( + org.apache.spark.sql.connector.catalog.Identifier ident, + org.apache.spark.sql.connector.catalog.Table table) { + refreshCount.incrementAndGet(); + } + }; + + assertThatThrownBy( + () -> + procedure.call( + new GenericInternalRow( + new Object[] { + UTF8String.fromString("db.t"), + UTF8String.fromString("SYNC"), + false + }))) + .isSameAs(listFailure); + assertThat(listCount).hasValue(2); + assertThat(partitionCatalog.createdPartitions).isEmpty(); + assertThat(partitionCatalog.droppedPartitions).isEmpty(); + // Every non-preview attempt refreshes the Spark-side caches, even a failed one. + assertThat(refreshCount).hasValue(1); + assertThat(partitionDirectory).exists(); + } finally { + spark.stop(); + SparkSession.clearActiveSession(); + SparkSession.clearDefaultSession(); + } + } + + @Test + void syncProcedureCallFailsClosedWhenFilesystemDiscoveryFailsPartway() throws Exception { + Files.createDirectories(tempDir.resolve("dt=20260715/hh=00")); + Files.createDirectories(tempDir.resolve("dt=20260716/hh=00")); + + IOException listFailure = new IOException("injected nested filesystem LIST failure"); + AtomicInteger completedPartitionListings = new AtomicInteger(); + FileIO fileIO = + new LocalFileIO() { + @Override + public FileStatus[] listStatus(Path path) throws IOException { + if ("dt=20260716".equals(path.getName())) { + throw listFailure; + } + FileStatus[] statuses = super.listStatus(path); + Arrays.sort( + statuses, + java.util.Comparator.comparing( + status -> status.getPath().toString())); + if ("dt=20260715".equals(path.getName())) { + completedPartitionListings.incrementAndGet(); + } + return statuses; + } + }; + Map options = new LinkedHashMap<>(); + options.put(CoreOptions.METASTORE_PARTITIONED_TABLE.key(), "true"); + RowType rowType = + RowType.builder() + .field("id", DataTypes.INT()) + .field("dt", DataTypes.STRING()) + .field("hh", DataTypes.STRING()) + .build(); + FormatTable partiallyListedTable = + FormatTable.builder() + .fileIO(fileIO) + .identifier(Identifier.create("db", "t")) + .rowType(rowType) + .partitionKeys(Arrays.asList("dt", "hh")) + .location(new Path(tempDir.toUri().toString()).toString()) + .format(FormatTable.Format.CSV) + .options(options) + .build(); + + SparkSession spark = + SparkSession.builder() + .master("local[1]") + .appName("sync-format-table-metadata-filesystem-failure-test") + .config("spark.ui.enabled", "false") + .getOrCreate(); + try { + AtomicInteger catalogListCount = new AtomicInteger(); + RecordingCatalog partitionCatalog = + new RecordingCatalog() { + @Override + public FormatTablePartitionPage listPartitions( + Map prefix, String pageToken, int pageSize) { + catalogListCount.incrementAndGet(); + return super.listPartitions(prefix, pageToken, pageSize); + } + }; + AtomicInteger refreshCount = new AtomicInteger(); + Procedure procedure = + new SyncFormatTableMetadataProcedure( + tableCatalog( + new PaimonFormatTable( + partiallyListedTable, partitionCatalog))) { + @Override + protected void refreshSparkCache( + org.apache.spark.sql.connector.catalog.Identifier ident, + org.apache.spark.sql.connector.catalog.Table table) { + refreshCount.incrementAndGet(); + } + }; + + assertThatThrownBy( + () -> + procedure.call( + new GenericInternalRow( + new Object[] { + UTF8String.fromString("db.t"), + UTF8String.fromString("SYNC"), + false + }))) + .isInstanceOf(RuntimeException.class) + .hasCause(listFailure); + assertThat(completedPartitionListings).hasValue(1); + assertThat(catalogListCount).hasValue(0); + assertThat(partitionCatalog.createdPartitions).isEmpty(); + assertThat(partitionCatalog.droppedPartitions).isEmpty(); + // Every non-preview attempt refreshes the Spark-side caches, even a failed one. + assertThat(refreshCount).hasValue(1); + } finally { + spark.stop(); + SparkSession.clearActiveSession(); + SparkSession.clearDefaultSession(); + } + } + + @Test + void syncRejectsUnsafeValueOnlyDirectoryBeforeCatalogMutation() throws Exception { + Files.createDirectories(tempDir.resolve("%2E%2E")); + SparkSession spark = + SparkSession.builder() + .master("local[1]") + .appName("sync-format-table-metadata-value-only-safety-test") + .config("spark.ui.enabled", "false") + .getOrCreate(); + try { + AtomicInteger catalogListCount = new AtomicInteger(); + RecordingCatalog partitionCatalog = + new RecordingCatalog() { + @Override + public FormatTablePartitionPage listPartitions( + Map prefix, String pageToken, int pageSize) { + catalogListCount.incrementAndGet(); + return super.listPartitions(prefix, pageToken, pageSize); + } + }; + Procedure procedure = + new SyncFormatTableMetadataProcedure( + tableCatalog( + new PaimonFormatTable( + formatTable(tempDir.toUri().toString(), true), + partitionCatalog))) { + @Override + protected void refreshSparkCache( + org.apache.spark.sql.connector.catalog.Identifier ident, + org.apache.spark.sql.connector.catalog.Table table) {} + }; + + assertThatThrownBy( + () -> + procedure.call( + new GenericInternalRow( + new Object[] { + UTF8String.fromString("db.t"), + UTF8String.fromString("SYNC"), + false + }))) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining(".."); + assertThat(catalogListCount).hasValue(0); + assertThat(partitionCatalog.createdPartitions).isEmpty(); + assertThat(partitionCatalog.droppedPartitions).isEmpty(); + } finally { + spark.stop(); + SparkSession.clearActiveSession(); + SparkSession.clearDefaultSession(); + } + } + + @Test + void syncTreatsEmptyNextPageTokenAsTerminal() { + RecordingCatalog catalog = new RecordingCatalog(); + catalog.addPage( + null, + new FormatTablePartitionPage( + Collections.singletonList(spec("dt", "20260701")), "")); + + List, String>> actions = + SyncFormatTableMetadataProcedure.syncPartitions( + catalog, + Arrays.asList(spec("dt", "20260701"), spec("dt", "20260702")), + Collections.singletonList("dt"), + true, + "ADD"); + + assertThat(actions).containsExactly(action(spec("dt", "20260702"), "ADD")); + assertThat(catalog.requestedTokens).containsExactly((String) null); + } + + @Test + void syncWithNoDiffDoesNotIssueAnEmptyCreate() { + RecordingCatalog catalog = new RecordingCatalog(); + catalog.addPage( + null, + new FormatTablePartitionPage( + Collections.singletonList(spec("dt", "20260701")), null)); + + List, String>> actions = + SyncFormatTableMetadataProcedure.syncPartitions( + catalog, + Collections.singletonList(spec("dt", "20260701")), + Collections.singletonList("dt"), + false, + "SYNC"); + + assertThat(actions).isEmpty(); + assertThat(catalog.createdPartitions).isEmpty(); + assertThat(catalog.droppedPartitions).isEmpty(); + } + + @Test + void syncRejectsUnsupportedMode() { + assertThatThrownBy(() -> SyncFormatTableMetadataProcedure.validateMode("RECOVER")) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("ADD, DROP and SYNC"); + SyncFormatTableMetadataProcedure.validateMode("ADD"); + SyncFormatTableMetadataProcedure.validateMode("drop"); + SyncFormatTableMetadataProcedure.validateMode("Sync"); + } + + @Test + void syncDropModeUnregistersOnlyMissingDirectories() { + RecordingCatalog catalog = new RecordingCatalog(); + catalog.addPage( + null, + new FormatTablePartitionPage( + Arrays.asList(spec("dt", "20260714"), spec("dt", "20260715")), null)); + + // dt=20260715 exists on the filesystem, dt=20260714 does not: only the latter is dropped. + List, String>> dryRun = + SyncFormatTableMetadataProcedure.syncPartitions( + catalog, + Collections.singletonList(spec("dt", "20260715")), + Collections.singletonList("dt"), + true, + "DROP"); + assertThat(dryRun).containsExactly(action(spec("dt", "20260714"), "DROP")); + assertThat(catalog.droppedPartitions).isEmpty(); + + List, String>> applied = + SyncFormatTableMetadataProcedure.syncPartitions( + catalog, + Collections.singletonList(spec("dt", "20260715")), + Collections.singletonList("dt"), + false, + "DROP"); + assertThat(applied).containsExactly(action(spec("dt", "20260714"), "DROP")); + assertThat(catalog.droppedPartitions) + .containsExactly(Collections.singletonList(spec("dt", "20260714"))); + assertThat(catalog.createdPartitions).isEmpty(); + } + + @Test + void syncDropModeAppliesLargeDiffInBoundedBatches() { + List> firstPage = new ArrayList<>(); + for (int index = 0; index < 1000; index++) { + firstPage.add(spec("dt", String.format("%04d", index))); + } + + RecordingCatalog catalog = new RecordingCatalog(); + catalog.addPage(null, new FormatTablePartitionPage(firstPage, "next")); + catalog.addPage( + "next", + new FormatTablePartitionPage(Collections.singletonList(spec("dt", "1000")), null)); + + List, String>> applied = + SyncFormatTableMetadataProcedure.syncPartitions( + catalog, + Collections.emptyList(), + Collections.singletonList("dt"), + false, + "DROP"); + + assertThat(applied).hasSize(1001); + assertThat(catalog.droppedPartitions).hasSize(2); + assertThat(catalog.droppedPartitions.get(0)).hasSize(1000); + assertThat(catalog.droppedPartitions.get(1)).hasSize(1); + assertThat(catalog.droppedPartitions.stream().mapToInt(List::size).sum()).isEqualTo(1001); + } + + @Test + void syncSyncModeAddsAndDropsInOneCall() { + RecordingCatalog catalog = new RecordingCatalog(); + catalog.addPage( + null, + new FormatTablePartitionPage( + Collections.singletonList(spec("dt", "20260714")), null)); + + List, String>> applied = + SyncFormatTableMetadataProcedure.syncPartitions( + catalog, + Collections.singletonList(spec("dt", "20260715")), + Collections.singletonList("dt"), + false, + "SYNC"); + assertThat(applied) + .containsExactly( + action(spec("dt", "20260715"), "ADD"), + action(spec("dt", "20260714"), "DROP")); + assertThat(catalog.createdPartitions) + .containsExactly(Collections.singletonList(spec("dt", "20260715"))); + assertThat(catalog.droppedPartitions) + .containsExactly(Collections.singletonList(spec("dt", "20260714"))); + } + + private static Map.Entry, String> action( + Map spec, String action) { + return new java.util.AbstractMap.SimpleEntry<>(spec, action); + } + + private static Map spec(String key, String value) { + Map spec = new LinkedHashMap<>(); + spec.put(key, value); + return spec; + } + + private static FormatTable formatTable() { + return formatTable("file:/tmp/format-table-procedure-test"); + } + + private static FormatTable formatTable(String location) { + return formatTable(location, false); + } + + private static FormatTable formatTable(String location, boolean onlyValueInPath) { + Map options = new LinkedHashMap<>(); + options.put(CoreOptions.METASTORE_PARTITIONED_TABLE.key(), "true"); + options.put( + CoreOptions.FORMAT_TABLE_PARTITION_ONLY_VALUE_IN_PATH.key(), + Boolean.toString(onlyValueInPath)); + RowType rowType = + RowType.builder() + .field("id", DataTypes.INT()) + .field("dt", DataTypes.STRING()) + .build(); + return FormatTable.builder() + .fileIO(LocalFileIO.create()) + .identifier(Identifier.create("db", "t")) + .rowType(rowType) + .partitionKeys(Collections.singletonList("dt")) + .location(new Path(location).toString()) + .format(FormatTable.Format.CSV) + .options(options) + .build(); + } + + private static TableCatalog tableCatalog(PaimonFormatTable table) { + return (TableCatalog) + Proxy.newProxyInstance( + FormatTableMetadataProcedureTest.class.getClassLoader(), + new Class[] {TableCatalog.class}, + (proxy, method, args) -> { + switch (method.getName()) { + case "name": + return "test_catalog"; + case "defaultNamespace": + return new String[] {"db"}; + case "loadTable": + return table; + case "equals": + return proxy == args[0]; + case "hashCode": + return System.identityHashCode(proxy); + case "toString": + return "test_catalog"; + default: + throw new UnsupportedOperationException(method.getName()); + } + }); + } + + private static class RecordingCatalog implements FormatTablePartitionCatalog { + + private final Map pages = new LinkedHashMap<>(); + private final List requestedTokens = new ArrayList<>(); + private final List requestedPageSizes = new ArrayList<>(); + private final List> requestedPrefixes = new ArrayList<>(); + private final List>> createdPartitions = new ArrayList<>(); + private final List createIgnoreFlags = new ArrayList<>(); + private final List>> droppedPartitions = new ArrayList<>(); + + private void addPage(String token, FormatTablePartitionPage page) { + pages.put(token, page); + } + + @Override + public void createPartitions(List> partitions, boolean ignoreIfExists) { + createdPartitions.add(new ArrayList<>(partitions)); + createIgnoreFlags.add(ignoreIfExists); + } + + @Override + public void dropPartitions(List> partitions) { + droppedPartitions.add(new ArrayList<>(partitions)); + } + + @Override + public List> listPartitionsByNames( + List> partitions) { + throw new UnsupportedOperationException(); + } + + @Override + public FormatTablePartitionPage listPartitions( + Map prefix, String pageToken, int pageSize) { + requestedPrefixes.add(prefix == null ? null : new LinkedHashMap<>(prefix)); + requestedTokens.add(pageToken); + requestedPageSizes.add(pageSize); + return pages.get(pageToken); + } + } +} diff --git a/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/commands/PaimonShowFormatTablePartitionsExecTest.scala b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/commands/PaimonShowFormatTablePartitionsExecTest.scala new file mode 100644 index 000000000000..707f051e4f87 --- /dev/null +++ b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/commands/PaimonShowFormatTablePartitionsExecTest.scala @@ -0,0 +1,229 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.spark.commands + +import org.apache.paimon.catalog.{CatalogContext, Identifier} +import org.apache.paimon.fs.Path +import org.apache.paimon.fs.local.LocalFileIO +import org.apache.paimon.options.Options +import org.apache.paimon.spark.PaimonSparkTestBase +import org.apache.paimon.spark.format.{FormatTablePartitionCatalog, FormatTablePartitionPage, PaimonFormatTable} +import org.apache.paimon.table.FormatTable +import org.apache.paimon.types.DataTypes + +import org.apache.spark.sql.catalyst.analysis.ResolvedPartitionSpec +import org.apache.spark.sql.catalyst.expressions.{AttributeReference, GenericInternalRow} +import org.apache.spark.sql.execution.SparkPlan +import org.apache.spark.sql.types.StringType +import org.apache.spark.unsafe.types.UTF8String + +import java.nio.file.Files +import java.util.{Collections, LinkedHashMap, List => JList, Map => JMap} + +import scala.collection.JavaConverters._ +import scala.collection.mutable.ArrayBuffer + +class PaimonShowFormatTablePartitionsExecTest extends PaimonSparkTestBase { + + test("managed Format Table SHOW PARTITIONS has a bounded catalog executor") { + Class.forName("org.apache.paimon.spark.commands.PaimonShowFormatTablePartitionsExec") + } + + test("managed Format Table SHOW PARTITIONS exposes its default Driver cap") { + assert(PaimonShowFormatTablePartitionsExec.DEFAULT_MAX_RESULTS == 10000) + } + + test("managed Format Table SHOW PARTITIONS pages catalog results and uses Spark formatting") { + val calls = ArrayBuffer.empty[(Map[String, String], String, Int)] + val gateway = new FormatTablePartitionCatalog { + override def createPartitions( + partitions: JList[JMap[String, String]], + ignoreIfExists: Boolean): Unit = unsupported() + + override def dropPartitions(partitions: JList[JMap[String, String]]): Unit = unsupported() + + override def listPartitionsByNames( + partitions: JList[JMap[String, String]]): JList[JMap[String, String]] = unsupported() + + override def listPartitions( + prefix: JMap[String, String], + pageToken: String, + pageSize: Int): FormatTablePartitionPage = { + calls += ((prefix.asScala.toMap, pageToken, pageSize)) + pageToken match { + case null => FormatTablePartitionPage(List(partition("z")).asJava, "page-2") + case "page-2" => FormatTablePartitionPage(List(partition("a=1")).asJava, null) + case token => fail(s"Unexpected page token $token") + } + } + } + val table = new PaimonFormatTable(managedFormatTable(), gateway) + val spec = ResolvedPartitionSpec( + Seq("dt"), + new GenericInternalRow(Array[Any](UTF8String.fromString("2026/07/15")))) + val output = Seq(AttributeReference("partition", StringType, nullable = false)()) + val constructor = classOf[PaimonShowFormatTablePartitionsExec].getConstructors + .find(_.getParameterCount == 4) + .getOrElse(fail("The catalog SHOW executor constructor is missing")) + val exec = constructor + .newInstance(output, table, Some(spec), Int.box(3)) + .asInstanceOf[SparkPlan] + + val actual = exec.executeCollect().map(_.getUTF8String(0).toString).toSeq + + assert(actual == Seq("dt=2026%2F07%2F15/region=a%3D1", "dt=2026%2F07%2F15/region=z")) + assert( + calls.toSeq == Seq( + (Map("dt" -> "2026/07/15"), null, 4), + (Map("dt" -> "2026/07/15"), "page-2", 3))) + } + + test("managed Format Table SHOW PARTITIONS rejects results above the Driver cap") { + val gateway = new FormatTablePartitionCatalog { + override def createPartitions( + partitions: JList[JMap[String, String]], + ignoreIfExists: Boolean): Unit = unsupported() + + override def dropPartitions(partitions: JList[JMap[String, String]]): Unit = unsupported() + + override def listPartitionsByNames( + partitions: JList[JMap[String, String]]): JList[JMap[String, String]] = unsupported() + + override def listPartitions( + prefix: JMap[String, String], + pageToken: String, + pageSize: Int): FormatTablePartitionPage = { + assert(pageSize == 3) + FormatTablePartitionPage(List(partition("a"), partition("b"), partition("c")).asJava, null) + } + } + val exec = PaimonShowFormatTablePartitionsExec( + Seq(AttributeReference("partition", StringType, nullable = false)()), + new PaimonFormatTable(managedFormatTable(), gateway), + None, + maxResults = 2) + + val error = intercept[RuntimeException] { + exec.executeCollect() + } + + assert(error.getMessage.contains("2")) + assert(error.getMessage.contains("CALL sys.list_format_table_partitions")) + } + + test("managed Format Table SHOW PARTITIONS continues after an empty catalog page") { + val tokens = ArrayBuffer.empty[String] + val gateway = new FormatTablePartitionCatalog { + override def createPartitions( + partitions: JList[JMap[String, String]], + ignoreIfExists: Boolean): Unit = unsupported() + + override def dropPartitions(partitions: JList[JMap[String, String]]): Unit = unsupported() + + override def listPartitionsByNames( + partitions: JList[JMap[String, String]]): JList[JMap[String, String]] = unsupported() + + override def listPartitions( + prefix: JMap[String, String], + pageToken: String, + pageSize: Int): FormatTablePartitionPage = { + tokens += pageToken + pageToken match { + case null => FormatTablePartitionPage(Collections.emptyList(), "page-2") + case "page-2" => FormatTablePartitionPage(List(partition("a")).asJava, null) + case token => fail(s"Unexpected page token $token") + } + } + } + val exec = PaimonShowFormatTablePartitionsExec( + Seq(AttributeReference("partition", StringType, nullable = false)()), + new PaimonFormatTable(managedFormatTable(), gateway), + None, + maxResults = 2) + + val actual = exec.executeCollect().map(_.getUTF8String(0).toString).toSeq + + assert(actual == Seq("dt=2026%2F07%2F15/region=a")) + assert(tokens.toSeq == Seq(null, "page-2")) + } + + test("managed Format Table SHOW PARTITIONS treats an empty next-page token as terminal") { + var calls = 0 + val gateway = new FormatTablePartitionCatalog { + override def createPartitions( + partitions: JList[JMap[String, String]], + ignoreIfExists: Boolean): Unit = unsupported() + + override def dropPartitions(partitions: JList[JMap[String, String]]): Unit = unsupported() + + override def listPartitionsByNames( + partitions: JList[JMap[String, String]]): JList[JMap[String, String]] = unsupported() + + override def listPartitions( + prefix: JMap[String, String], + pageToken: String, + pageSize: Int): FormatTablePartitionPage = { + calls += 1 + if (calls > 1) { + fail("An empty page token must not be fetched") + } + FormatTablePartitionPage(List(partition("a")).asJava, "") + } + } + val exec = PaimonShowFormatTablePartitionsExec( + Seq(AttributeReference("partition", StringType, nullable = false)()), + new PaimonFormatTable(managedFormatTable(), gateway), + None, + maxResults = 2) + + val actual = exec.executeCollect().map(_.getUTF8String(0).toString).toSeq + + assert(actual == Seq("dt=2026%2F07%2F15/region=a")) + assert(calls == 1) + } + + private def partition(region: String): JMap[String, String] = { + val result = new LinkedHashMap[String, String]() + result.put("region", region) + result.put("dt", "2026/07/15") + result + } + + private def managedFormatTable(): FormatTable = { + FormatTable + .builder() + .fileIO(LocalFileIO.create) + .identifier(Identifier.create("test_db", "format_table")) + .rowType( + DataTypes.ROW( + DataTypes.FIELD(0, "id", DataTypes.INT()), + DataTypes.FIELD(1, "dt", DataTypes.STRING()), + DataTypes.FIELD(2, "region", DataTypes.STRING()))) + .partitionKeys(Seq("dt", "region").asJava) + .location(new Path(Files.createTempDirectory("format_table_show").toUri).toString) + .format(FormatTable.Format.CSV) + .options(Map("metastore.partitioned-table" -> "true").asJava) + .catalogContext(CatalogContext.create(new Options)) + .build() + } + + private def unsupported[T](): T = { + throw new UnsupportedOperationException("Not used by SHOW PARTITIONS") + } +} diff --git a/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/execution/FormatTablePartitionDdlPlanningTest.scala b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/execution/FormatTablePartitionDdlPlanningTest.scala index a98f2361f07a..71a14cec72db 100644 --- a/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/execution/FormatTablePartitionDdlPlanningTest.scala +++ b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/execution/FormatTablePartitionDdlPlanningTest.scala @@ -24,6 +24,7 @@ import org.apache.paimon.fs.local.LocalFileIO import org.apache.paimon.options.Options import org.apache.paimon.spark.PaimonSparkTestWithRestCatalogBase import org.apache.paimon.spark.catalyst.plans.logical.PaimonDropPartitions +import org.apache.paimon.spark.commands.PaimonShowFormatTablePartitionsExec import org.apache.paimon.spark.format.{FormatTablePartitionCatalog, FormatTablePartitionPage, PaimonFormatTable} import org.apache.paimon.table.FormatTable import org.apache.paimon.types.DataTypes @@ -31,9 +32,10 @@ import org.apache.paimon.types.DataTypes import org.apache.spark.sql.AnalysisException import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.catalyst.analysis.{NoSuchPartitionsException, ResolvedPartitionSpec, ResolvedTable} -import org.apache.spark.sql.catalyst.expressions.GenericInternalRow -import org.apache.spark.sql.catalyst.plans.logical.{AddPartitions, DropPartitions} +import org.apache.spark.sql.catalyst.expressions.{AttributeReference, GenericInternalRow} +import org.apache.spark.sql.catalyst.plans.logical.{AddPartitions, DropPartitions, RepairTable, ShowPartitions} import org.apache.spark.sql.connector.catalog.{Identifier => SparkIdentifier, TableCatalog} +import org.apache.spark.sql.types.StringType import java.io.IOException import java.lang.reflect.InvocationTargetException @@ -242,7 +244,7 @@ class FormatTablePartitionDdlPlanningTest extends PaimonSparkTestWithRestCatalog // Leading partial spec expands to every registered leaf partition below it. sql(s"ALTER TABLE $tableName DROP PARTITION (dt='20260715')") - assert(catalogPartitions(tableName) == Set("dt=20260716/hh=10")) + assert(shownPartitions(tableName) == Set("dt=20260716/hh=10")) // Complete specs honor IF EXISTS and fail loudly for unregistered partitions. sql(s"ALTER TABLE $tableName DROP IF EXISTS PARTITION (dt='20990101', hh='00')") @@ -254,11 +256,11 @@ class FormatTablePartitionDdlPlanningTest extends PaimonSparkTestWithRestCatalog .iterate(missingError: Throwable)(_.getCause) .takeWhile(_ != null) .exists(_.isInstanceOf[NoSuchPartitionsException])) - assert(catalogPartitions(tableName) == Set("dt=20260716/hh=10")) + assert(shownPartitions(tableName) == Set("dt=20260716/hh=10")) // Non-leading partial specs resolve by partition name through the gateway. sql(s"ALTER TABLE $tableName DROP PARTITION (hh='10')") - assert(catalogPartitions(tableName) == Set.empty) + assert(shownPartitions(tableName) == Set.empty) } } @@ -360,7 +362,7 @@ class FormatTablePartitionDdlPlanningTest extends PaimonSparkTestWithRestCatalog () => refreshCalls += 1)) } - // The unregistered directory (still awaiting registration) must survive untouched. + // The unregistered directory (e.g. awaiting MSCK registration) must survive untouched. assert(dropCalls == 0) assert(refreshCalls == 0) assert(fileIO.exists(pendingDir)) @@ -809,6 +811,67 @@ class FormatTablePartitionDdlPlanningTest extends PaimonSparkTestWithRestCatalog assert(dropCalls == 0) } + test("MSCK strategy intercepts only managed Format Tables and maps the mode flags") { + val catalog = spark.sessionState.catalogManager.currentCatalog.asInstanceOf[TableCatalog] + val identifier = SparkIdentifier.of(Array("test"), "format_table") + val (managed, _) = formatTable(managed = true) + val (unmanaged, _) = formatTable(managed = false) + val managedResolved = ResolvedTable.create(catalog, identifier, managed) + + // (plain MSCK) -> ADD; DROP PARTITIONS -> DROP; SYNC PARTITIONS -> both. + Seq((true, false), (false, true), (true, true)).foreach { + case (add, drop) => + val plans = PaimonStrategy(spark).apply(RepairTable(managedResolved, add, drop)) + assert(plans.size == 1) + val repair = plans.head.asInstanceOf[PaimonRepairFormatTablePartitionsExec] + assert(repair.addPartitions == add) + assert(repair.dropPartitions == drop) + } + + // Unmanaged tables keep Spark's own v2 rejection: no interception. + val unmanagedPlans = PaimonStrategy(spark) + .apply(RepairTable(ResolvedTable.create(catalog, identifier, unmanaged), true, false)) + assert(unmanagedPlans.isEmpty) + } + + test("MSCK repair reuses the sync engine: DROP unregisters catalog-only partitions") { + val gateway = new AtomicGateway(Set(Map("dt" -> "20260715", "hh" -> "10"))) + val table = new PaimonFormatTable(rawFormatTable(managed = true), gateway) + var refreshCalls = 0 + + // The table location has no partition directories, so the registered partition is + // catalog-only; MSCK DROP PARTITIONS must unregister it (metadata-only). + runCommand( + PaimonRepairFormatTablePartitionsExec( + table, + addPartitions = false, + dropPartitions = true, + () => refreshCalls += 1)) + + assert(gateway.state.isEmpty) + assert(refreshCalls == 1) + } + + test("strategy routes only managed Format Table SHOW PARTITIONS through the bounded executor") { + val output = Seq(AttributeReference("partition", StringType, nullable = false)()) + val (managed, _) = formatTable(managed = true) + val (unmanaged, _) = formatTable(managed = false) + val catalog = spark.sessionState.catalogManager.currentCatalog.asInstanceOf[TableCatalog] + val identifier = SparkIdentifier.of(Array("test"), "format_table") + val spec = Some(partition(20260715, 10)) + + val managedPlans = PaimonStrategy(spark).apply( + ShowPartitions(ResolvedTable.create(catalog, identifier, managed), spec, output)) + val unmanagedPlans = PaimonStrategy(spark).apply( + ShowPartitions(ResolvedTable.create(catalog, identifier, unmanaged), spec, output)) + + assert(managedPlans.size == 1) + assert( + managedPlans.head.asInstanceOf[PaimonShowFormatTablePartitionsExec].maxResults == + PaimonShowFormatTablePartitionsExec.DEFAULT_MAX_RESULTS) + assert(unmanagedPlans.isEmpty) + } + private def partition(dt: Int, hh: Int): ResolvedPartitionSpec = ResolvedPartitionSpec(Seq("dt", "hh"), new GenericInternalRow(Array[Any](dt, hh))) @@ -855,12 +918,8 @@ class FormatTablePartitionDdlPlanningTest extends PaimonSparkTestWithRestCatalog specs.map(_.asJava).asJava, true) - private def catalogPartitions(tableName: String): Set[String] = - paimonCatalog - .listPartitions(Identifier.create(dbName0, tableName)) - .asScala - .map(p => Seq("dt", "hh").map(key => s"$key=${p.spec().get(key)}").mkString("/")) - .toSet + private def shownPartitions(tableName: String): Set[String] = + sql(s"SHOW PARTITIONS $tableName").collect().map(_.getString(0)).toSet private def runCommand(command: AnyRef): Unit = { try { diff --git a/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/ManagedFormatTableMsckRepairTest.scala b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/ManagedFormatTableMsckRepairTest.scala new file mode 100644 index 000000000000..ddcefa8820a3 --- /dev/null +++ b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/ManagedFormatTableMsckRepairTest.scala @@ -0,0 +1,602 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.spark.sql + +import org.apache.paimon.catalog.Identifier +import org.apache.paimon.fs.Path +import org.apache.paimon.spark.{PaimonSparkTestWithRestCatalogBase, SparkCatalog} +import org.apache.paimon.spark.execution.PaimonRepairFormatTablePartitionsExec +import org.apache.paimon.spark.format.{FormatTablePartitionCatalog, FormatTablePartitionPage, PaimonFormatTable} +import org.apache.paimon.table.FormatTable + +import org.apache.spark.SparkConf +import org.apache.spark.sql.Row +import org.apache.spark.sql.connector.catalog.{Identifier => SparkIdentifier, Table => SparkTable} +import org.apache.spark.sql.execution.CommandResultExec + +import java.lang.reflect.InvocationTargetException +import java.util.{List => JList, Map => JMap} + +import scala.collection.JavaConverters._ +import scala.collection.mutable + +class ManagedFormatTableMsckRepairTest extends PaimonSparkTestWithRestCatalogBase { + + override protected def sparkConf: SparkConf = + super.sparkConf + .set("spark.sql.catalog.paimon", classOf[FaultInjectingFormatTableSparkCatalog].getName) + + override protected def beforeEach(): Unit = { + MsckFaultInjection.reset() + super.beforeEach() + } + + override protected def afterEach(): Unit = { + try { + super.afterEach() + } finally { + MsckFaultInjection.reset() + } + } + + test("bare MSCK, explicit ADD, and REPAIR alias register filesystem-only partitions") { + val tableName = "msck_add_forms" + val first = "20260715" + val second = "20260716" + val third = "20260717" + + withTable(tableName) { + createManagedFormatTable(tableName) + + writeCsvPartition(tableName, first, 15, "bare-msck") + executeManagedRepair(s"MSCK REPAIR TABLE paimon.$dbName0.$tableName") + assertPartitionState(tableName, Set(first)) + + writeCsvPartition(tableName, second, 16, "explicit-add") + executeManagedRepair(s"MSCK REPAIR TABLE paimon.$dbName0.$tableName ADD PARTITIONS") + assertPartitionState(tableName, Set(first, second)) + + writeCsvPartition(tableName, third, 17, "repair-alias") + executeManagedRepair(s"REPAIR TABLE paimon.$dbName0.$tableName ADD PARTITIONS") + assertPartitionState(tableName, Set(first, second, third)) + + executeManagedRepair(s"REPAIR TABLE paimon.$dbName0.$tableName ADD PARTITIONS") + + assertPartitionState(tableName, Set(first, second, third)) + assert(MsckFaultInjection.createCalls == 3) + assertRowsAndChecksum( + tableName, + Seq( + Row(15, "bare-msck", first), + Row(16, "explicit-add", second), + Row(17, "repair-alias", third)), + 48L) + } + } + + test("DROP unregisters only catalog-only partitions and preserves valid files") { + val tableName = "msck_drop_catalog_only" + val catalogOnly = "20260714" + val common = "20260715" + + withTable(tableName) { + createManagedFormatTable(tableName) + val commonPath = writeCsvPartition(tableName, common, 15, "common") + registerPartitions(tableName, catalogOnly, common) + + executeManagedRepair(s"MSCK REPAIR TABLE paimon.$dbName0.$tableName DROP PARTITIONS") + + assertPartitionState(tableName, Set(common)) + assert(formatTable(tableName).fileIO().exists(commonPath)) + assertRowsAndChecksum(tableName, Seq(Row(15, "common", common)), 15L) + + executeManagedRepair(s"MSCK REPAIR TABLE paimon.$dbName0.$tableName DROP PARTITIONS") + assertPartitionState(tableName, Set(common)) + assert(MsckFaultInjection.createCalls == 0) + assert(MsckFaultInjection.dropCalls == 1) + } + } + + test("SYNC converges both directions and is idempotent") { + val tableName = "msck_sync_both_directions" + val catalogOnly = "20260714" + val filesystemOnly = "20260715" + val common = "20260716" + + withTable(tableName) { + createManagedFormatTable(tableName) + val filesystemPath = + writeCsvPartition(tableName, filesystemOnly, 15, "filesystem-only") + val commonPath = writeCsvPartition(tableName, common, 16, "common") + registerPartitions(tableName, catalogOnly, common) + + executeManagedRepair(s"MSCK REPAIR TABLE paimon.$dbName0.$tableName SYNC PARTITIONS") + + assertPartitionState(tableName, Set(filesystemOnly, common)) + assert(formatTable(tableName).fileIO().exists(filesystemPath)) + assert(formatTable(tableName).fileIO().exists(commonPath)) + assertRowsAndChecksum( + tableName, + Seq(Row(15, "filesystem-only", filesystemOnly), Row(16, "common", common)), + 31L) + assert(MsckFaultInjection.createCalls == 1) + assert(MsckFaultInjection.dropCalls == 1) + + executeManagedRepair(s"MSCK REPAIR TABLE paimon.$dbName0.$tableName SYNC PARTITIONS") + assertPartitionState(tableName, Set(filesystemOnly, common)) + assert(MsckFaultInjection.createCalls == 1) + assert(MsckFaultInjection.dropCalls == 1) + } + } + + test("MSCK rejects unmanaged Format Tables and native Paimon tables") { + val unmanaged = "msck_unmanaged_format_table" + val native = "msck_native_paimon_table" + val partition = "20260715" + + withTable(unmanaged, native) { + createUnmanagedFormatTable(unmanaged) + writeCsvPartition(unmanaged, partition, 15, "unmanaged") + checkAnswer( + sql(s"SELECT id, payload, dt FROM $unmanaged"), + Seq(Row(15, "unmanaged", partition))) + + assertMsckRejected(s"MSCK REPAIR TABLE paimon.$dbName0.$unmanaged") + checkAnswer( + sql(s"SELECT id, payload, dt FROM $unmanaged"), + Seq(Row(15, "unmanaged", partition))) + + sql( + s"CREATE TABLE $native (id INT, payload STRING, dt STRING) " + + "USING paimon PARTITIONED BY (dt)") + sql(s"INSERT INTO $native VALUES (16, 'native', '$partition')") + + assertMsckRejected(s"MSCK REPAIR TABLE paimon.$dbName0.$native") + checkAnswer(sql(s"SELECT id, payload, dt FROM $native"), Seq(Row(16, "native", partition))) + assert(MsckFaultInjection.createCalls == 0) + assert(MsckFaultInjection.dropCalls == 0) + } + } + + test("SYNC propagates catalog LIST failure without mutating either side") { + val tableName = "msck_sync_list_failure" + val catalogOnly = "20260714" + val filesystemOnly = "20260715" + + withTable(tableName) { + createManagedFormatTable(tableName) + val filesystemPath = + writeCsvPartition(tableName, filesystemOnly, 15, "filesystem-only") + registerPartitions(tableName, catalogOnly) + MsckFaultInjection.failList = true + + val error = intercept[Exception] { + sql(s"MSCK REPAIR TABLE paimon.$dbName0.$tableName SYNC PARTITIONS").collect() + } + + assert(causeMessages(error).contains(MsckFaultInjection.LIST_FAILURE)) + assert(MsckFaultInjection.listCalls == 1) + assert(MsckFaultInjection.createCalls == 0) + assert(MsckFaultInjection.dropCalls == 0) + assert(registeredPartitions(tableName) == Set(catalogOnly)) + assert(formatTable(tableName).fileIO().exists(filesystemPath)) + + MsckFaultInjection.failList = false + executeManagedRepair(s"MSCK REPAIR TABLE paimon.$dbName0.$tableName SYNC PARTITIONS") + assertPartitionState(tableName, Set(filesystemOnly)) + assertRowsAndChecksum(tableName, Seq(Row(15, "filesystem-only", filesystemOnly)), 15L) + } + } + + test("direct repair mutates nothing but still refreshes when catalog LIST fails") { + val tableName = "msck_direct_list_failure" + + withTable(tableName) { + createManagedFormatTable(tableName) + val gateway = new StatefulFaultCatalog + gateway.failList = true + var refreshCalls = 0 + val command = PaimonRepairFormatTablePartitionsExec( + new PaimonFormatTable(formatTable(tableName), gateway), + addPartitions = true, + dropPartitions = true, + () => refreshCalls += 1) + + val error = intercept[IllegalStateException] { + runCommand(command) + } + + assert(error.getMessage == MsckFaultInjection.LIST_FAILURE) + assert(gateway.createCalls == 0) + assert(gateway.dropCalls == 0) + // A failed response cannot prove the service did nothing, so every repair attempt + // refreshes the Spark-side caches. + assert(refreshCalls == 1) + } + } + + test("direct repair refreshes after create applies and loses its response") { + val tableName = "msck_direct_create_lost_response" + val filesystemOnly = "20260715" + + withTable(tableName) { + createManagedFormatTable(tableName) + writeCsvPartition(tableName, filesystemOnly, 15, "filesystem-only") + val gateway = new StatefulFaultCatalog + gateway.failCreateAfterApply = true + var refreshCalls = 0 + val command = PaimonRepairFormatTablePartitionsExec( + new PaimonFormatTable(formatTable(tableName), gateway), + addPartitions = true, + dropPartitions = true, + () => refreshCalls += 1) + + val error = intercept[IllegalStateException] { + runCommand(command) + } + + assert(error.getMessage == MsckFaultInjection.CREATE_LOST_RESPONSE) + assert(gateway.partitions == Set(Map("dt" -> filesystemOnly))) + assert(gateway.createCalls == 1) + assert(gateway.dropCalls == 0) + assert(refreshCalls == 1) + } + } + + test("direct repair still attempts refresh when DROP fails after a durable ADD") { + val tableName = "msck_direct_drop_and_refresh_failure" + val catalogOnly = "20260714" + val filesystemOnly = "20260715" + + withTable(tableName) { + createManagedFormatTable(tableName) + writeCsvPartition(tableName, filesystemOnly, 15, "filesystem-only") + val gateway = new StatefulFaultCatalog(Set(Map("dt" -> catalogOnly))) + gateway.failDrop = true + var refreshCalls = 0 + val command = PaimonRepairFormatTablePartitionsExec( + new PaimonFormatTable(formatTable(tableName), gateway), + addPartitions = true, + dropPartitions = true, + () => { + refreshCalls += 1 + throw new IllegalStateException(MsckFaultInjection.REFRESH_FAILURE) + } + ) + + val error = intercept[IllegalStateException] { + runCommand(command) + } + + // The refresh sits in a plain finally, so its failure is the one that propagates; the + // point is that the ADD stayed durable and the refresh was still attempted after the + // DROP failure. + assert(error.getMessage == MsckFaultInjection.REFRESH_FAILURE) + assert(gateway.partitions == Set(Map("dt" -> catalogOnly), Map("dt" -> filesystemOnly))) + assert(gateway.createCalls == 1) + assert(gateway.dropCalls == 1) + assert(refreshCalls == 1) + } + } + + test("direct repair preserves a DROP throwable rethrown by refresh") { + val tableName = "msck_direct_same_drop_refresh_failure" + val catalogOnly = "20260714" + val filesystemOnly = "20260715" + + withTable(tableName) { + createManagedFormatTable(tableName) + writeCsvPartition(tableName, filesystemOnly, 15, "filesystem-only") + val gateway = new StatefulFaultCatalog(Set(Map("dt" -> catalogOnly))) + val dropFailure = new IllegalStateException(MsckFaultInjection.DROP_FAILURE) + gateway.dropFailure = dropFailure + var refreshCalls = 0 + val command = PaimonRepairFormatTablePartitionsExec( + new PaimonFormatTable(formatTable(tableName), gateway), + addPartitions = true, + dropPartitions = true, + () => { + refreshCalls += 1 + throw dropFailure + } + ) + + val error = intercept[Throwable] { + runCommand(command) + } + + assert(error eq dropFailure) + assert(!error.isInstanceOf[MatchError]) + assert(error.getSuppressed.isEmpty) + assert(gateway.partitions == Set(Map("dt" -> catalogOnly), Map("dt" -> filesystemOnly))) + assert(gateway.createCalls == 1) + assert(gateway.dropCalls == 1) + assert(refreshCalls == 1) + } + } + + test("SYNC invalidates cached data and converges after ADD succeeds but DROP fails") { + val tableName = "msck_partial_sync_failure" + val filesystemOnly = "20260715" + val catalogOnly = "20260714" + + withTable(tableName) { + createManagedFormatTable(tableName) + sql(s"CACHE TABLE $tableName").collect() + assert(spark.catalog.isCached(tableName)) + checkAnswer(sql(s"SELECT * FROM $tableName"), Seq.empty[Row]) + + writeCsvPartition(tableName, filesystemOnly, 15, "filesystem-only") + registerPartitions(tableName, catalogOnly) + MsckFaultInjection.failDrop = true + + val error = intercept[Exception] { + sql(s"MSCK REPAIR TABLE paimon.$dbName0.$tableName SYNC PARTITIONS").collect() + } + + assert(causeMessages(error).contains(MsckFaultInjection.DROP_FAILURE)) + assert(MsckFaultInjection.createCalls == 1) + assert(MsckFaultInjection.dropCalls == 1) + assert(registeredPartitions(tableName) == Set(filesystemOnly, catalogOnly)) + assert(spark.catalog.isCached(tableName)) + + // ADD is already durable even though the command failed. A stale CACHE TABLE must not hide it. + checkAnswer( + sql(s"SELECT id, payload, dt FROM $tableName WHERE dt = '$filesystemOnly'"), + Seq(Row(15, "filesystem-only", filesystemOnly))) + + MsckFaultInjection.failDrop = false + executeManagedRepair(s"MSCK REPAIR TABLE paimon.$dbName0.$tableName SYNC PARTITIONS") + + assertPartitionState(tableName, Set(filesystemOnly)) + assert(MsckFaultInjection.createCalls == 1) + assert(MsckFaultInjection.dropCalls == 2) + assertRowsAndChecksum(tableName, Seq(Row(15, "filesystem-only", filesystemOnly)), 15L) + + executeManagedRepair(s"MSCK REPAIR TABLE paimon.$dbName0.$tableName SYNC PARTITIONS") + assertPartitionState(tableName, Set(filesystemOnly)) + assert(MsckFaultInjection.createCalls == 1) + assert(MsckFaultInjection.dropCalls == 2) + } + } + + private def createManagedFormatTable(tableName: String): Unit = + sql(s"""CREATE TABLE $tableName (id INT, payload STRING, dt STRING) + |USING CSV + |PARTITIONED BY (dt) + |TBLPROPERTIES ( + | 'format-table.implementation' = 'paimon', + | 'metastore.partitioned-table' = 'true') + |""".stripMargin) + + private def createUnmanagedFormatTable(tableName: String): Unit = + sql(s"""CREATE TABLE $tableName (id INT, payload STRING, dt STRING) + |USING CSV + |PARTITIONED BY (dt) + |TBLPROPERTIES ( + | 'format-table.implementation' = 'paimon', + | 'metastore.partitioned-table' = 'false') + |""".stripMargin) + + private def executeManagedRepair(statement: String): Unit = { + val result = sql(statement) + val resultPlan = result.queryExecution.executedPlan + assert(resultPlan.isInstanceOf[CommandResultExec], resultPlan.treeString) + val commandPlan = resultPlan.asInstanceOf[CommandResultExec].commandPhysicalPlan + assert(commandPlan.isInstanceOf[PaimonRepairFormatTablePartitionsExec], commandPlan.treeString) + result.collect() + } + + private def assertMsckRejected(statement: String): Unit = { + val error = intercept[Exception] { + sql(statement).collect() + } + val messages = causeMessages(error) + assert(messages.contains("MSCK REPAIR TABLE"), messages) + assert(messages.toLowerCase(java.util.Locale.ROOT).contains("not supported"), messages) + } + + private def assertPartitionState(tableName: String, expected: Set[String]): Unit = { + assert(registeredPartitions(tableName) == expected) + assert( + sql(s"SHOW PARTITIONS $tableName").collect().map(_.getString(0)).toSet == + expected.map(value => s"dt=$value")) + } + + private def assertRowsAndChecksum( + tableName: String, + expectedRows: Seq[Row], + expectedIdSum: Long): Unit = { + checkAnswer(sql(s"SELECT id, payload, dt FROM $tableName ORDER BY id"), expectedRows) + checkAnswer( + sql(s"SELECT COUNT(*), COALESCE(SUM(id), 0) FROM $tableName"), + Seq(Row(expectedRows.size.toLong, expectedIdSum))) + } + + private def runCommand(command: AnyRef): Unit = { + try { + command.getClass.getMethod("run").invoke(command) + } catch { + case error: InvocationTargetException => throw error.getCause + } + } + + private def tableIdentifier(tableName: String): Identifier = + Identifier.create(dbName0, tableName) + + private def formatTable(tableName: String): FormatTable = + paimonCatalog.getTable(tableIdentifier(tableName)).asInstanceOf[FormatTable] + + private def writeCsvPartition( + tableName: String, + partition: String, + id: Int, + payload: String): Path = { + val table = formatTable(tableName) + val partitionPath = new Path(table.location(), s"dt=$partition") + table.fileIO().mkdirs(partitionPath) + table.fileIO().writeFile(new Path(partitionPath, f"part-$id%05d.csv"), s"$id,$payload\n", false) + partitionPath + } + + private def registerPartitions(tableName: String, partitions: String*): Unit = + paimonCatalog.createPartitions( + tableIdentifier(tableName), + partitions.map(value => Map("dt" -> value).asJava).asJava, + true) + + private def registeredPartitions(tableName: String): Set[String] = + paimonCatalog + .listPartitions(tableIdentifier(tableName)) + .asScala + .map(_.spec().get("dt")) + .toSet + + private def causeMessages(error: Throwable): String = + Iterator + .iterate(error)(_.getCause) + .takeWhile(_ != null) + .flatMap(cause => Option(cause.getMessage)) + .mkString(" | ") +} + +private[sql] object MsckFaultInjection { + val DROP_FAILURE = "injected MSCK partition drop failure" + val LIST_FAILURE = "injected MSCK partition list failure" + val CREATE_LOST_RESPONSE = "injected MSCK create lost response" + val REFRESH_FAILURE = "injected MSCK cache refresh failure" + + @volatile var failDrop = false + @volatile var failList = false + @volatile var createCalls = 0 + @volatile var dropCalls = 0 + @volatile var listCalls = 0 + + def reset(): Unit = { + failDrop = false + failList = false + createCalls = 0 + dropCalls = 0 + listCalls = 0 + } +} + +private[sql] class StatefulFaultCatalog(initial: Set[Map[String, String]] = Set.empty) + extends FormatTablePartitionCatalog { + + private val state = mutable.LinkedHashSet(initial.toSeq: _*) + var failList = false + var failCreateAfterApply = false + var failDrop = false + var dropFailure: RuntimeException = null + var createCalls = 0 + var dropCalls = 0 + + def partitions: Set[Map[String, String]] = state.toSet + + override def createPartitions( + partitions: JList[JMap[String, String]], + ignoreIfExists: Boolean): Unit = { + createCalls += 1 + state ++= partitions.asScala.map(_.asScala.toMap) + if (failCreateAfterApply) { + throw new IllegalStateException(MsckFaultInjection.CREATE_LOST_RESPONSE) + } + } + + override def dropPartitions(partitions: JList[JMap[String, String]]): Unit = { + dropCalls += 1 + if (dropFailure != null) { + throw dropFailure + } + if (failDrop) { + throw new IllegalStateException(MsckFaultInjection.DROP_FAILURE) + } + state --= partitions.asScala.map(_.asScala.toMap) + } + + override def listPartitionsByNames( + partitions: JList[JMap[String, String]]): JList[JMap[String, String]] = + partitions.asScala + .map(_.asScala.toMap) + .filter(state.contains) + .map(_.asJava) + .asJava + + override def listPartitions( + prefix: JMap[String, String], + pageToken: String, + pageSize: Int): FormatTablePartitionPage = { + if (failList) { + throw new IllegalStateException(MsckFaultInjection.LIST_FAILURE) + } + if (pageToken != null) { + throw new AssertionError(s"Unexpected page token $pageToken") + } + FormatTablePartitionPage(state.toSeq.map(_.asJava).asJava, null) + } +} + +/** Spark-loadable test catalog which delegates all state to the embedded Paimon REST catalog. */ +private[sql] class FaultInjectingFormatTableSparkCatalog extends SparkCatalog { + + override def loadTable(ident: SparkIdentifier): SparkTable = { + super.loadTable(ident) match { + case table: PaimonFormatTable if table.partitionCatalog != null => + new PaimonFormatTable( + table.table, + new FaultInjectingFormatTablePartitionCatalog(table.partitionCatalog)) + case table => table + } + } +} + +private[sql] class FaultInjectingFormatTablePartitionCatalog(delegate: FormatTablePartitionCatalog) + extends FormatTablePartitionCatalog { + + override def createPartitions( + partitions: JList[JMap[String, String]], + ignoreIfExists: Boolean): Unit = { + delegate.createPartitions(partitions, ignoreIfExists) + MsckFaultInjection.createCalls += 1 + } + + override def dropPartitions(partitions: JList[JMap[String, String]]): Unit = { + MsckFaultInjection.dropCalls += 1 + if (MsckFaultInjection.failDrop) { + throw new IllegalStateException(MsckFaultInjection.DROP_FAILURE) + } + delegate.dropPartitions(partitions) + } + + override def listPartitionsByNames( + partitions: JList[JMap[String, String]]): JList[JMap[String, String]] = + delegate.listPartitionsByNames(partitions) + + override def listPartitions( + prefix: JMap[String, String], + pageToken: String, + pageSize: Int): FormatTablePartitionPage = { + MsckFaultInjection.listCalls += 1 + if (MsckFaultInjection.failList) { + throw new IllegalStateException(MsckFaultInjection.LIST_FAILURE) + } + delegate.listPartitions(prefix, pageToken, pageSize) + } +}