diff --git a/docs/docs/primary-key-table/global-index.mdx b/docs/docs/primary-key-table/global-index.mdx index ba59a1e972a6..3098d71c52e4 100644 --- a/docs/docs/primary-key-table/global-index.mdx +++ b/docs/docs/primary-key-table/global-index.mdx @@ -234,8 +234,6 @@ schema validation. | `fields..pk-btree.index.options` | Not set | JSON object containing BTree build options. Unqualified keys are scoped to `btree-index`. | | `pk-bitmap.index.columns` | Not set | Comma-separated columns which own independent Bitmap indexes. | | `fields..pk-bitmap.index.options` | Not set | JSON object containing Bitmap build options. Unqualified keys are scoped to `bitmap-index`. | -| `fields..pk-index.compaction.level-fanout` | `5` | Number of similarly sized index groups which triggers a rebuild and maximum row-count ratio within one size tier. Shared by all four families. Must be greater than `1`. | -| `fields..pk-index.compaction.stale-ratio-threshold` | `0.2` | Ratio of rows from inactive source files which triggers a rebuild. Shared by all four families. Must be in `(0, 1]`. | | `global-index.search-mode` | `fast` | Search mode for primary-key Vector and Full Text queries. `fast` searches indexed data only, so uncovered files are omitted. For Vector, `full` and `detail` search uncovered files exactly. Primary-key Full Text supports only `fast`. | For algorithm-specific options, see the corresponding @@ -259,17 +257,16 @@ inside real buckets. These rows become visible after batch compaction publishes buckets. Indexes are created when that process physically rewrites the rows into eligible compact output; simply assigning or upgrading a pending file does not make it an index source. -### Index LSM Maintenance +### Data-Level Maintenance -Each indexed column maintains its own immutable index groups. Maintenance uses the shared -field-scoped compaction options: +Each indexed column maintains one immutable index payload for the complete eligible source-file +set in every non-zero data level. When data compaction changes a level, Paimon rebuilds that whole +level payload, including files in the target level which were not direct compaction inputs. A +level payload is used only when its ordered source names and row counts exactly match the current +data level; partial, duplicate, stale, and cross-level payloads are rejected. -- When at least `level-fanout` similarly sized groups exist, Paimon rebuilds them into a larger - group. The largest selected group can contain at most `level-fanout` times the rows of the - smallest group. -- When the ratio of rows belonging to inactive source files reaches - `stale-ratio-threshold`, Paimon rebuilds the affected group from its remaining active sources. -- A rebuild atomically replaces its input groups after the new group is complete. +A rebuild atomically replaces the old payload after the complete new payload is ready. Unrelated +data levels retain their existing payloads. Index construction can execute asynchronously inside the writer. A writer which waits for compaction also waits for active index maintenance; a non-blocking writer can complete maintenance diff --git a/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java b/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java index 271e46346e99..4f1cb3111193 100644 --- a/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java +++ b/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java @@ -4299,22 +4299,6 @@ public boolean primaryKeyFullTextIndexEnabled() { return options.getOptional(PK_FULL_TEXT_INDEX_COLUMNS).isPresent(); } - public int primaryKeyIndexCompactionLevelFanout(String column) { - return options.getInteger(primaryKeyIndexCompactionLevelFanoutKey(column), 5); - } - - public double primaryKeyIndexCompactionStaleRatioThreshold(String column) { - return options.getDouble(primaryKeyIndexCompactionStaleRatioThresholdKey(column), 0.2); - } - - public static String primaryKeyIndexCompactionLevelFanoutKey(String column) { - return "fields." + column + ".pk-index.compaction.level-fanout"; - } - - public static String primaryKeyIndexCompactionStaleRatioThresholdKey(String column) { - return "fields." + column + ".pk-index.compaction.stale-ratio-threshold"; - } - public List primaryKeyVectorIndexColumns() { return primaryKeyIndexColumns(PK_VECTOR_INDEX_COLUMNS); } diff --git a/paimon-core/src/main/java/org/apache/paimon/index/pk/BucketedPrimaryKeyIndexMaintainer.java b/paimon-core/src/main/java/org/apache/paimon/index/pk/BucketedPrimaryKeyIndexMaintainer.java index 27faa7774f09..b9a9223c1119 100644 --- a/paimon-core/src/main/java/org/apache/paimon/index/pk/BucketedPrimaryKeyIndexMaintainer.java +++ b/paimon-core/src/main/java/org/apache/paimon/index/pk/BucketedPrimaryKeyIndexMaintainer.java @@ -381,9 +381,7 @@ public static Factory create( readerFactoryBuilder, field, definition.indexType(), - definition.options(), - definition.compactionLevelFanout(), - definition.compactionStaleRatioThreshold())); + definition.options())); break; case FULL_TEXT: checkArgument( @@ -391,11 +389,7 @@ public static Factory create( "Only one primary-key full-text index is supported."); fullTextFactory = new FullTextDefinitionFactory( - readerFactoryBuilder, - field, - definition.options(), - definition.compactionLevelFanout(), - definition.compactionStaleRatioThreshold()); + readerFactoryBuilder, field, definition.options()); break; default: throw new IllegalArgumentException( @@ -480,20 +474,14 @@ private static final class FullTextDefinitionFactory { private final KeyValueFileReaderFactory.Builder readerFactoryBuilder; private final DataField field; private final org.apache.paimon.options.Options options; - private final int compactionLevelFanout; - private final double compactionStaleRatioThreshold; private FullTextDefinitionFactory( KeyValueFileReaderFactory.Builder readerFactoryBuilder, DataField field, - org.apache.paimon.options.Options options, - int compactionLevelFanout, - double compactionStaleRatioThreshold) { + org.apache.paimon.options.Options options) { this.readerFactoryBuilder = readerFactoryBuilder; this.field = field; this.options = options; - this.compactionLevelFanout = compactionLevelFanout; - this.compactionStaleRatioThreshold = compactionStaleRatioThreshold; } private BucketedFullTextIndexMaintainer create( @@ -515,8 +503,6 @@ private BucketedFullTextIndexMaintainer create( field.id(), indexFile, builder, - compactionLevelFanout, - compactionStaleRatioThreshold, restoredDataFiles, restoredPayloads, executor); @@ -529,22 +515,16 @@ private static final class SortedDefinitionFactory { private final DataField field; private final String indexType; private final org.apache.paimon.options.Options options; - private final int compactionLevelFanout; - private final double compactionStaleRatioThreshold; private SortedDefinitionFactory( KeyValueFileReaderFactory.Builder readerFactoryBuilder, DataField field, String indexType, - org.apache.paimon.options.Options options, - int compactionLevelFanout, - double compactionStaleRatioThreshold) { + org.apache.paimon.options.Options options) { this.readerFactoryBuilder = readerFactoryBuilder; this.field = field; this.indexType = indexType; this.options = options; - this.compactionLevelFanout = compactionLevelFanout; - this.compactionStaleRatioThreshold = compactionStaleRatioThreshold; } private BucketedSortedIndexMaintainer create( @@ -570,8 +550,6 @@ private BucketedSortedIndexMaintainer create( indexType, indexFile, builder::build, - compactionLevelFanout, - compactionStaleRatioThreshold, restoredDataFiles, restoredPayloads, executor); diff --git a/paimon-core/src/main/java/org/apache/paimon/index/pk/PrimaryKeyIndexDefinition.java b/paimon-core/src/main/java/org/apache/paimon/index/pk/PrimaryKeyIndexDefinition.java index f5d47f8b605f..f04ea4389a4b 100644 --- a/paimon-core/src/main/java/org/apache/paimon/index/pk/PrimaryKeyIndexDefinition.java +++ b/paimon-core/src/main/java/org/apache/paimon/index/pk/PrimaryKeyIndexDefinition.java @@ -36,24 +36,14 @@ public enum Family { private final String indexType; private final Options options; private final Family family; - private final int compactionLevelFanout; - private final double compactionStaleRatioThreshold; public PrimaryKeyIndexDefinition( - String column, - int fieldId, - String indexType, - Options options, - Family family, - int compactionLevelFanout, - double compactionStaleRatioThreshold) { + String column, int fieldId, String indexType, Options options, Family family) { this.column = column; this.fieldId = fieldId; this.indexType = indexType; this.options = options; this.family = family; - this.compactionLevelFanout = compactionLevelFanout; - this.compactionStaleRatioThreshold = compactionStaleRatioThreshold; } public String column() { @@ -75,12 +65,4 @@ public Options options() { public Family family() { return family; } - - public int compactionLevelFanout() { - return compactionLevelFanout; - } - - public double compactionStaleRatioThreshold() { - return compactionStaleRatioThreshold; - } } diff --git a/paimon-core/src/main/java/org/apache/paimon/index/pk/PrimaryKeyIndexDefinitions.java b/paimon-core/src/main/java/org/apache/paimon/index/pk/PrimaryKeyIndexDefinitions.java index 0ac0353e047e..9b7bdcdfedfe 100644 --- a/paimon-core/src/main/java/org/apache/paimon/index/pk/PrimaryKeyIndexDefinitions.java +++ b/paimon-core/src/main/java/org/apache/paimon/index/pk/PrimaryKeyIndexDefinitions.java @@ -63,9 +63,7 @@ public static PrimaryKeyIndexDefinitions create(TableSchema schema) { field.id(), BTreeGlobalIndexerFactory.IDENTIFIER, options.primaryKeyBTreeIndexOptions(column), - PrimaryKeyIndexDefinition.Family.BTREE, - options.primaryKeyIndexCompactionLevelFanout(column), - options.primaryKeyIndexCompactionStaleRatioThreshold(column))); + PrimaryKeyIndexDefinition.Family.BTREE)); } else if (bitmapColumns.contains(column)) { definitions.add( new PrimaryKeyIndexDefinition( @@ -73,9 +71,7 @@ public static PrimaryKeyIndexDefinitions create(TableSchema schema) { field.id(), BitmapGlobalIndexerFactory.IDENTIFIER, options.primaryKeyBitmapIndexOptions(column), - PrimaryKeyIndexDefinition.Family.BITMAP, - options.primaryKeyIndexCompactionLevelFanout(column), - options.primaryKeyIndexCompactionStaleRatioThreshold(column))); + PrimaryKeyIndexDefinition.Family.BITMAP)); } else if (vectorColumns.contains(column)) { definitions.add( new PrimaryKeyIndexDefinition( @@ -83,9 +79,7 @@ public static PrimaryKeyIndexDefinitions create(TableSchema schema) { field.id(), options.primaryKeyVectorIndexType(column), options.primaryKeyVectorIndexOptions(column), - PrimaryKeyIndexDefinition.Family.VECTOR, - options.primaryKeyIndexCompactionLevelFanout(column), - options.primaryKeyIndexCompactionStaleRatioThreshold(column))); + PrimaryKeyIndexDefinition.Family.VECTOR)); } else if (fullTextColumns.contains(column)) { definitions.add( new PrimaryKeyIndexDefinition( @@ -93,9 +87,7 @@ public static PrimaryKeyIndexDefinitions create(TableSchema schema) { field.id(), "full-text", options.primaryKeyFullTextIndexOptions(column), - PrimaryKeyIndexDefinition.Family.FULL_TEXT, - options.primaryKeyIndexCompactionLevelFanout(column), - options.primaryKeyIndexCompactionStaleRatioThreshold(column))); + PrimaryKeyIndexDefinition.Family.FULL_TEXT)); } } diff --git a/paimon-core/src/main/java/org/apache/paimon/index/pk/PrimaryKeyIndexLevels.java b/paimon-core/src/main/java/org/apache/paimon/index/pk/PrimaryKeyIndexLevels.java index ff3dae74b07c..7002d6c47e36 100644 --- a/paimon-core/src/main/java/org/apache/paimon/index/pk/PrimaryKeyIndexLevels.java +++ b/paimon-core/src/main/java/org/apache/paimon/index/pk/PrimaryKeyIndexLevels.java @@ -26,7 +26,9 @@ import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.Set; import java.util.TreeMap; +import java.util.TreeSet; import java.util.function.Function; import static org.apache.paimon.utils.Preconditions.checkArgument; @@ -34,125 +36,110 @@ /** Derives logical compaction levels from immutable primary-key index source metadata. */ public final class PrimaryKeyIndexLevels { - private final int fanout; - private final double staleRatioThreshold; - private final Function identity; + private final Function dataLevel; private final Function> sources; public PrimaryKeyIndexLevels( - int fanout, - double staleRatioThreshold, - Function identity, - Function> sources) { - checkArgument(fanout > 1, "Primary-key index level fanout must be greater than one."); - checkArgument( - staleRatioThreshold > 0 && staleRatioThreshold <= 1, - "Primary-key index stale ratio threshold must be in (0, 1]."); - this.fanout = fanout; - this.staleRatioThreshold = staleRatioThreshold; - this.identity = identity; + Function dataLevel, Function> sources) { + this.dataLevel = dataLevel; this.sources = sources; } public Optional> pick(List units, Map activeSourceFiles) { - T staleCandidate = null; - double highestStaleRatio = -1; - for (T unit : units) { - double staleRatio = staleRatio(unit, activeSourceFiles); - if (staleRatio >= staleRatioThreshold - && (staleRatio > highestStaleRatio - || (staleRatio == highestStaleRatio - && (staleCandidate == null - || identity.apply(unit) - .compareTo( - identity.apply(staleCandidate)) - < 0)))) { - staleCandidate = unit; - highestStaleRatio = staleRatio; + Map> desiredByLevel = new TreeMap<>(); + for (DataFileMeta file : activeSourceFiles.values()) { + if (file.level() > 0) { + desiredByLevel + .computeIfAbsent(file.level(), ignored -> new ArrayList<>()) + .add(file); } } - if (staleCandidate != null) { - return Optional.of( - createPlan(Collections.singletonList(staleCandidate), activeSourceFiles)); + for (List files : desiredByLevel.values()) { + files.sort(Comparator.comparing(DataFileMeta::fileName)); + } + + Map unitsByLevel = new TreeMap<>(); + for (T unit : units) { + int level = dataLevel.apply(unit); + checkArgument(level > 0, "Primary-key index data level must be positive."); + checkArgument( + unitsByLevel.put(level, unit) == null, + "Multiple primary-key index units exist for data level %s.", + level); } - List candidates = new ArrayList<>(units); - candidates.sort( - Comparator.comparingLong(this::buildRowCount).thenComparing(identity::apply)); - for (int start = 0; start + fanout <= candidates.size(); start++) { - long smallest = buildRowCount(candidates.get(start)); - long largest = buildRowCount(candidates.get(start + fanout - 1)); - if (largest <= saturatedMultiply(smallest, fanout)) { - return Optional.of( - createPlan( - new ArrayList<>(candidates.subList(start, start + fanout)), - activeSourceFiles)); + Set levels = new TreeSet<>(); + levels.addAll(desiredByLevel.keySet()); + levels.addAll(unitsByLevel.keySet()); + for (int level : levels) { + T unit = unitsByLevel.get(level); + List desired = + desiredByLevel.getOrDefault(level, Collections.emptyList()); + if (unit == null) { + return Optional.of(new Plan<>(level, Collections.emptyList(), desired)); + } + if (!matches(sources.apply(unit), desired)) { + return Optional.of(new Plan<>(level, Collections.singletonList(unit), desired)); } } return Optional.empty(); } - private double staleRatio(T unit, Map activeSourceFiles) { - long totalRows = 0; - long staleRows = 0; - for (PrimaryKeyIndexSourceFile source : sources.apply(unit)) { - totalRows = Math.addExact(totalRows, source.rowCount()); - DataFileMeta active = activeSourceFiles.get(source.fileName()); - if (active == null) { - staleRows = Math.addExact(staleRows, source.rowCount()); - } else { - checkArgument( - active.rowCount() == source.rowCount(), - "Primary-key index source %s row count does not match active data file.", - source.fileName()); + public boolean isCurrent(Plan plan, Map activeSourceFiles) { + List current = new ArrayList<>(); + for (DataFileMeta file : activeSourceFiles.values()) { + if (file.level() == plan.dataLevel()) { + current.add(file); } } - return totalRows == 0 ? 0 : ((double) staleRows) / totalRows; - } - - private Plan createPlan(List inputUnits, Map activeSourceFiles) { - Map selectedSources = new TreeMap<>(); - for (T unit : inputUnits) { - for (PrimaryKeyIndexSourceFile source : sources.apply(unit)) { - DataFileMeta active = activeSourceFiles.get(source.fileName()); - if (active != null) { - checkArgument( - active.rowCount() == source.rowCount(), - "Primary-key index source %s row count does not match active data file.", - source.fileName()); - selectedSources.put(active.fileName(), active); - } + current.sort(Comparator.comparing(DataFileMeta::fileName)); + if (plan.sourceFiles().size() != current.size()) { + return false; + } + for (int i = 0; i < current.size(); i++) { + DataFileMeta planned = plan.sourceFiles().get(i); + DataFileMeta actual = current.get(i); + if (!planned.fileName().equals(actual.fileName()) + || planned.rowCount() != actual.rowCount()) { + return false; } } - return new Plan<>(inputUnits, new ArrayList<>(selectedSources.values())); + return true; } - private long buildRowCount(T unit) { - long rowCount = 0; - for (PrimaryKeyIndexSourceFile source : sources.apply(unit)) { - rowCount = Math.addExact(rowCount, source.rowCount()); + private static boolean matches( + List sources, List desired) { + if (sources.size() != desired.size()) { + return false; } - return rowCount; - } - - private static long saturatedMultiply(long value, int multiplier) { - if (value > Long.MAX_VALUE / multiplier) { - return Long.MAX_VALUE; + for (int i = 0; i < sources.size(); i++) { + PrimaryKeyIndexSourceFile source = sources.get(i); + DataFileMeta file = desired.get(i); + if (!source.fileName().equals(file.fileName()) + || source.rowCount() != file.rowCount()) { + return false; + } } - return value * multiplier; + return true; } /** A deterministic primary-key index rebuild selection. */ public static final class Plan { + private final int dataLevel; private final List inputUnits; private final List sourceFiles; - private Plan(List inputUnits, List sourceFiles) { + private Plan(int dataLevel, List inputUnits, List sourceFiles) { + this.dataLevel = dataLevel; this.inputUnits = Collections.unmodifiableList(new ArrayList<>(inputUnits)); this.sourceFiles = Collections.unmodifiableList(new ArrayList<>(sourceFiles)); } + public int dataLevel() { + return dataLevel; + } + public List inputUnits() { return inputUnits; } diff --git a/paimon-core/src/main/java/org/apache/paimon/index/pk/PrimaryKeyIndexSourceMeta.java b/paimon-core/src/main/java/org/apache/paimon/index/pk/PrimaryKeyIndexSourceMeta.java index 072829e96c04..0b26b621674d 100644 --- a/paimon-core/src/main/java/org/apache/paimon/index/pk/PrimaryKeyIndexSourceMeta.java +++ b/paimon-core/src/main/java/org/apache/paimon/index/pk/PrimaryKeyIndexSourceMeta.java @@ -35,15 +35,22 @@ public final class PrimaryKeyIndexSourceMeta { private static final int VERSION = 1; + private final int dataLevel; private final List sourceFiles; - public PrimaryKeyIndexSourceMeta(List sourceFiles) { + public PrimaryKeyIndexSourceMeta(int dataLevel, List sourceFiles) { + checkArgument(dataLevel > 0, "Primary-key index data level must be positive."); + this.dataLevel = dataLevel; this.sourceFiles = Collections.unmodifiableList(new ArrayList<>(sourceFiles)); checkArgument(!this.sourceFiles.isEmpty(), "An index must reference source files."); } - public PrimaryKeyIndexSourceMeta(PrimaryKeyIndexSourceFile sourceFile) { - this(Collections.singletonList(sourceFile)); + public PrimaryKeyIndexSourceMeta(int dataLevel, PrimaryKeyIndexSourceFile sourceFile) { + this(dataLevel, Collections.singletonList(sourceFile)); + } + + public int dataLevel() { + return dataLevel; } public List sourceFiles() { @@ -71,6 +78,7 @@ public byte[] serialize() { try { DataOutputSerializer output = new DataOutputSerializer(128); output.writeInt(VERSION); + output.writeInt(dataLevel); output.writeInt(sourceFiles.size()); for (PrimaryKeyIndexSourceFile sourceFile : sourceFiles) { output.writeUTF(sourceFile.fileName()); @@ -87,6 +95,7 @@ public static PrimaryKeyIndexSourceMeta deserialize(byte[] bytes) { DataInputDeserializer input = new DataInputDeserializer(bytes); int version = input.readInt(); checkArgument(version == VERSION, "Unsupported index source version: %s.", version); + int dataLevel = input.readInt(); int sourceFileCount = input.readInt(); checkArgument(sourceFileCount > 0, "An index must reference source files."); // Each entry needs at least the two-byte writeUTF length and one long. @@ -104,7 +113,7 @@ public static PrimaryKeyIndexSourceMeta deserialize(byte[] bytes) { } checkArgument( input.available() == 0, "Unexpected trailing bytes in index source metadata."); - return new PrimaryKeyIndexSourceMeta(sourceFiles); + return new PrimaryKeyIndexSourceMeta(dataLevel, sourceFiles); } catch (IOException e) { throw new IllegalArgumentException("Failed to deserialize index source metadata.", e); } diff --git a/paimon-core/src/main/java/org/apache/paimon/index/pkfulltext/BucketedFullTextIndexMaintainer.java b/paimon-core/src/main/java/org/apache/paimon/index/pkfulltext/BucketedFullTextIndexMaintainer.java index 9231c7897977..8b3f79c3ea63 100644 --- a/paimon-core/src/main/java/org/apache/paimon/index/pkfulltext/BucketedFullTextIndexMaintainer.java +++ b/paimon-core/src/main/java/org/apache/paimon/index/pkfulltext/BucketedFullTextIndexMaintainer.java @@ -31,7 +31,6 @@ import java.util.ArrayList; import java.util.Collections; -import java.util.Comparator; import java.util.HashSet; import java.util.LinkedHashMap; import java.util.List; @@ -45,12 +44,9 @@ import static org.apache.paimon.utils.Preconditions.checkArgument; -/** Maintains bucket-local full-text archives with source-backed LSM consolidation. */ +/** Maintains one full-text archive for each complete bucket-local data level. */ public class BucketedFullTextIndexMaintainer { - private static final int DEFAULT_LEVEL_FANOUT = 5; - private static final double DEFAULT_STALE_RATIO_THRESHOLD = 0.2; - private final int textFieldId; private final PkFullTextIndexFile indexFile; private final PkFullTextIndexBuilder indexBuilder; @@ -68,34 +64,12 @@ public BucketedFullTextIndexMaintainer( List restoredDataFiles, List restoredPayloads, ExecutorService executor) { - this( - textFieldId, - indexFile, - indexBuilder, - DEFAULT_LEVEL_FANOUT, - DEFAULT_STALE_RATIO_THRESHOLD, - restoredDataFiles, - restoredPayloads, - executor); - } - - public BucketedFullTextIndexMaintainer( - int textFieldId, - PkFullTextIndexFile indexFile, - PkFullTextIndexBuilder indexBuilder, - int levelFanout, - double staleRatioThreshold, - List restoredDataFiles, - List restoredPayloads, - ExecutorService executor) { this.textFieldId = textFieldId; this.indexFile = indexFile; this.indexBuilder = indexBuilder; this.levels = new PrimaryKeyIndexLevels<>( - levelFanout, - staleRatioThreshold, - IndexFileMeta::fileName, + payload -> sourceMeta(payload).dataLevel(), payload -> sourceMeta(payload).sourceFiles()); this.executor = executor; for (DataFileMeta file : restoredDataFiles) { @@ -105,7 +79,8 @@ public BucketedFullTextIndexMaintainer( } PkFullTextBucketIndexState restoredState = - PkFullTextBucketIndexState.fromActivePayloads(textFieldId, restoredPayloads); + PkFullTextBucketIndexState.fromActiveDataFiles( + textFieldId, new ArrayList<>(activeSourceFiles.values()), restoredPayloads); currentPayloads.addAll(restoredState.currentPayloads()); retiredPayloads.addAll(restoredState.stalePayloads()); validateActiveSourceRows(); @@ -144,20 +119,14 @@ public synchronized FullTextIndexCommit prepareCommit( } if (pendingBuild == null) { - List uncovered = uncoveredFiles(); - if (!uncovered.isEmpty()) { - startBuild(uncovered, Collections.emptyList()); - } else { - Optional> plan = - levels.pick(currentPayloads, activeSourceFiles); - if (plan.isPresent()) { - if (plan.get().sourceFiles().isEmpty()) { - removePayloads( - plan.get().inputUnits(), created, removed, generated); - continue; - } - startBuild(plan.get().sourceFiles(), plan.get().inputUnits()); + Optional> plan = + levels.pick(currentPayloads, activeSourceFiles); + if (plan.isPresent()) { + if (plan.get().sourceFiles().isEmpty()) { + removePayloads(plan.get().inputUnits(), created, removed, generated); + continue; } + startBuild(plan.get()); } } if (!waitCompaction || pendingBuild == null) { @@ -212,20 +181,8 @@ private void applyDataTransition(CompactIncrement compactIncrement) { } } - private List uncoveredFiles() { - Set covered = coveredSources(currentPayloads); - List uncovered = new ArrayList<>(); - for (DataFileMeta source : activeSourceFiles.values()) { - if (!covered.contains(source.fileName())) { - uncovered.add(source); - } - } - uncovered.sort(Comparator.comparing(DataFileMeta::fileName)); - return uncovered; - } - - private void startBuild(List sourceFiles, List inputPayloads) { - PendingBuild build = new PendingBuild(sourceFiles, inputPayloads); + private void startBuild(PrimaryKeyIndexLevels.Plan plan) { + PendingBuild build = new PendingBuild(plan); build.start(); pendingBuild = build; } @@ -238,8 +195,7 @@ private Optional finishPendingBuild(boolean blocking) throws Exc try { IndexFileMeta payload = completed.get(); pendingBuild = null; - return Optional.of( - new CompletedBuild(completed.sourceFiles, completed.inputPayloads, payload)); + return Optional.of(new CompletedBuild(completed.plan, payload)); } catch (CancellationException e) { pendingBuild = null; return Optional.empty(); @@ -257,16 +213,21 @@ private Optional finishPendingBuild(boolean blocking) throws Exc } private boolean canAccept(CompletedBuild build) { - if (!currentPayloads.containsAll(build.inputPayloads)) { + if (!levels.isCurrent(build.plan, activeSourceFiles) + || !currentPayloads.containsAll(build.inputPayloads)) { return false; } PkFullTextBucketIndexState outputState = - PkFullTextBucketIndexState.fromActivePayloads( - textFieldId, Collections.singletonList(build.payload)); + PkFullTextBucketIndexState.fromActiveDataFiles( + textFieldId, build.sourceFiles, Collections.singletonList(build.payload)); if (outputState.currentPayloads().size() != 1) { return false; } - List actualSources = sourceMeta(build.payload).sourceFiles(); + PrimaryKeyIndexSourceMeta actualSourceMeta = sourceMeta(build.payload); + if (actualSourceMeta.dataLevel() != build.plan.dataLevel()) { + return false; + } + List actualSources = actualSourceMeta.sourceFiles(); if (actualSources.size() != build.sourceFiles.size()) { return false; } @@ -370,7 +331,8 @@ public synchronized void close() { } public synchronized PkFullTextBucketIndexState state() { - return PkFullTextBucketIndexState.fromActivePayloads(textFieldId, currentPayloads); + return PkFullTextBucketIndexState.fromActiveDataFiles( + textFieldId, new ArrayList<>(activeSourceFiles.values()), currentPayloads); } public synchronized List payloads() { @@ -393,15 +355,17 @@ private void validateActiveSourceRows() { private class PendingBuild { + private final PrimaryKeyIndexLevels.Plan plan; private final List sourceFiles; private final List inputPayloads; @Nullable private IndexFileMeta result; @Nullable private Future future; private boolean cancelled; - private PendingBuild(List sourceFiles, List inputPayloads) { - this.sourceFiles = Collections.unmodifiableList(new ArrayList<>(sourceFiles)); - this.inputPayloads = Collections.unmodifiableList(new ArrayList<>(inputPayloads)); + private PendingBuild(PrimaryKeyIndexLevels.Plan plan) { + this.plan = plan; + this.sourceFiles = plan.sourceFiles(); + this.inputPayloads = plan.inputUnits(); } private void start() { @@ -451,16 +415,16 @@ private void cancel() { private static final class CompletedBuild { + private final PrimaryKeyIndexLevels.Plan plan; private final List sourceFiles; private final List inputPayloads; private final IndexFileMeta payload; private CompletedBuild( - List sourceFiles, - List inputPayloads, - IndexFileMeta payload) { - this.sourceFiles = sourceFiles; - this.inputPayloads = inputPayloads; + PrimaryKeyIndexLevels.Plan plan, IndexFileMeta payload) { + this.plan = plan; + this.sourceFiles = plan.sourceFiles(); + this.inputPayloads = plan.inputUnits(); this.payload = payload; } } diff --git a/paimon-core/src/main/java/org/apache/paimon/index/pkfulltext/PkFullTextBucketIndexState.java b/paimon-core/src/main/java/org/apache/paimon/index/pkfulltext/PkFullTextBucketIndexState.java index 01daa9f5e3ea..ad9f5fddf89d 100644 --- a/paimon-core/src/main/java/org/apache/paimon/index/pkfulltext/PkFullTextBucketIndexState.java +++ b/paimon-core/src/main/java/org/apache/paimon/index/pkfulltext/PkFullTextBucketIndexState.java @@ -22,14 +22,18 @@ import org.apache.paimon.index.IndexFileMeta; import org.apache.paimon.index.pk.PrimaryKeyIndexSourceFile; import org.apache.paimon.index.pk.PrimaryKeyIndexSourceMeta; +import org.apache.paimon.index.pk.PrimaryKeyIndexSourcePolicy; +import org.apache.paimon.io.DataFileMeta; import java.util.ArrayList; import java.util.Collections; +import java.util.Comparator; import java.util.HashSet; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Set; +import java.util.TreeMap; import static org.apache.paimon.utils.Preconditions.checkArgument; @@ -41,9 +45,82 @@ public final class PkFullTextBucketIndexState { private final List stalePayloads; private final Map payloadBySourceFile; - public static PkFullTextBucketIndexState fromActivePayloads( - int textFieldId, List activePayloads) { - return new PkFullTextBucketIndexState(textFieldId, activePayloads); + public static PkFullTextBucketIndexState fromActiveDataFiles( + int textFieldId, + List activeDataFiles, + List activePayloads) { + Map> sourcesByLevel = new TreeMap<>(); + for (DataFileMeta dataFile : activeDataFiles) { + if (PrimaryKeyIndexSourcePolicy.shouldRead(dataFile)) { + sourcesByLevel + .computeIfAbsent(dataFile.level(), ignored -> new ArrayList<>()) + .add( + new PrimaryKeyIndexSourceFile( + dataFile.fileName(), dataFile.rowCount())); + } + } + for (List sources : sourcesByLevel.values()) { + sources.sort(Comparator.comparing(PrimaryKeyIndexSourceFile::fileName)); + } + + Map> payloadsByLevel = new TreeMap<>(); + List stale = new ArrayList<>(); + for (IndexFileMeta payload : activePayloads) { + GlobalIndexMeta globalMeta = payload.globalIndexMeta(); + if (!PkFullTextIndexFile.INDEX_TYPE.equals(payload.indexType()) || globalMeta == null) { + continue; + } + if (globalMeta.indexFieldId() != textFieldId) { + if (globalMeta.sourceMeta() != null) { + stale.add(payload); + } + continue; + } + try { + PrimaryKeyIndexSourceMeta sourceMeta = + PrimaryKeyIndexSourceMeta.fromIndexFile(payload); + List desired = + sourcesByLevel.get(sourceMeta.dataLevel()); + PkFullTextBucketIndexState singleton = + new PkFullTextBucketIndexState( + textFieldId, Collections.singletonList(payload)); + if (desired == null + || !desired.equals(sourceMeta.sourceFiles()) + || singleton.currentPayloads().size() != 1) { + stale.add(payload); + } else { + payloadsByLevel + .computeIfAbsent(sourceMeta.dataLevel(), ignored -> new ArrayList<>()) + .add(payload); + } + } catch (RuntimeException ignored) { + stale.add(payload); + } + } + + List current = new ArrayList<>(); + for (List levelPayloads : payloadsByLevel.values()) { + if (levelPayloads.size() == 1) { + current.add(levelPayloads.get(0)); + } else { + stale.addAll(levelPayloads); + } + } + PkFullTextBucketIndexState state = new PkFullTextBucketIndexState(textFieldId, current); + return new PkFullTextBucketIndexState( + textFieldId, state.currentPayloads, stale, state.payloadBySourceFile); + } + + private PkFullTextBucketIndexState( + int textFieldId, + List currentPayloads, + List stalePayloads, + Map payloadBySourceFile) { + this.textFieldId = textFieldId; + this.currentPayloads = Collections.unmodifiableList(new ArrayList<>(currentPayloads)); + this.stalePayloads = Collections.unmodifiableList(new ArrayList<>(stalePayloads)); + this.payloadBySourceFile = + Collections.unmodifiableMap(new LinkedHashMap<>(payloadBySourceFile)); } public PkFullTextBucketIndexState(int textFieldId, List activePayloads) { diff --git a/paimon-core/src/main/java/org/apache/paimon/index/pkfulltext/PkFullTextIndexFile.java b/paimon-core/src/main/java/org/apache/paimon/index/pkfulltext/PkFullTextIndexFile.java index 361b8f00d00d..1b2f22fddd23 100644 --- a/paimon-core/src/main/java/org/apache/paimon/index/pkfulltext/PkFullTextIndexFile.java +++ b/paimon-core/src/main/java/org/apache/paimon/index/pkfulltext/PkFullTextIndexFile.java @@ -97,9 +97,16 @@ IndexFileMeta build( IndexFileMeta build(List sources, DataField textField, GlobalIndexer indexer) throws IOException { checkArgument(!sources.isEmpty(), "A full-text archive must reference source files."); + int dataLevel = sources.get(0).sourceFile.level(); + checkArgument(dataLevel > 0, "A full-text archive requires a positive data level."); long totalRowCount = 0; List sourceFiles = new ArrayList<>(sources.size()); for (Source source : sources) { + checkArgument( + source.sourceFile.level() == dataLevel, + "A full-text archive cannot mix data levels %s and %s.", + dataLevel, + source.sourceFile.level()); checkArgument( source.sourceFile.rowCount() > 0, "Full-text source file must contain rows."); totalRowCount = Math.addExact(totalRowCount, source.sourceFile.rowCount()); @@ -163,7 +170,8 @@ IndexFileMeta build(List sources, DataField textField, GlobalIndexer ind textField.id(), null, archiveMetadata, - new PrimaryKeyIndexSourceMeta(sourceFiles).serialize()), + new PrimaryKeyIndexSourceMeta(dataLevel, sourceFiles) + .serialize()), pathFactory.isExternalPath() ? archivePath.toString() : null); success = true; return archive; diff --git a/paimon-core/src/main/java/org/apache/paimon/index/pksorted/BucketedSortedIndexMaintainer.java b/paimon-core/src/main/java/org/apache/paimon/index/pksorted/BucketedSortedIndexMaintainer.java index a0216da28bba..7c56779b0604 100644 --- a/paimon-core/src/main/java/org/apache/paimon/index/pksorted/BucketedSortedIndexMaintainer.java +++ b/paimon-core/src/main/java/org/apache/paimon/index/pksorted/BucketedSortedIndexMaintainer.java @@ -21,6 +21,7 @@ import org.apache.paimon.index.IndexFileMeta; import org.apache.paimon.index.pk.PrimaryKeyIndexLevels; import org.apache.paimon.index.pk.PrimaryKeyIndexSourceFile; +import org.apache.paimon.index.pk.PrimaryKeyIndexSourceMeta; import org.apache.paimon.index.pk.PrimaryKeyIndexSourcePolicy; import org.apache.paimon.io.CompactIncrement; import org.apache.paimon.io.DataFileMeta; @@ -33,7 +34,6 @@ import java.util.ArrayList; import java.util.Collections; -import java.util.Comparator; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -68,8 +68,6 @@ public BucketedSortedIndexMaintainer( String indexType, PkSortedIndexFile indexFile, BuildFunction buildFunction, - int levelFanout, - double staleRatioThreshold, List restoredDataFiles, List restoredPayloads, ExecutorService executor) { @@ -79,10 +77,7 @@ public BucketedSortedIndexMaintainer( this.buildFunction = buildFunction; this.levels = new PrimaryKeyIndexLevels<>( - levelFanout, - staleRatioThreshold, - PkSortedIndexGroup::identity, - PkSortedIndexGroup::sourceFiles); + PkSortedIndexGroup::dataLevel, PkSortedIndexGroup::sourceFiles); this.executor = executor; for (DataFileMeta dataFile : restoredDataFiles) { if (PrimaryKeyIndexSourcePolicy.shouldRead(dataFile)) { @@ -99,8 +94,11 @@ public BucketedSortedIndexMaintainer( } } PkSortedBucketIndexState restoredState = - PkSortedBucketIndexState.fromActivePayloads( - fieldId, indexType, sourceFiles(), definitionPayloads); + PkSortedBucketIndexState.fromActiveDataFiles( + fieldId, + indexType, + new ArrayList<>(activeSourceFiles.values()), + definitionPayloads); groups.addAll(restoredState.groups()); pendingRestoredDeletions.addAll(restoredState.rejectedPayloads()); } @@ -142,23 +140,15 @@ public synchronized SortedIndexCommit prepareCommit( } if (pendingBuild == null && allowBuildStart) { - DataFileMeta uncovered = firstUncoveredSource(); - if (uncovered != null) { - startBuild(Collections.singletonList(uncovered), Collections.emptyList()); - } else { - Optional> plan = - levels.pick(groups, activeSourceFiles); - if (plan.isPresent()) { - if (plan.get().sourceFiles().isEmpty()) { - replaceInputGroups( - plan.get().inputUnits(), - Optional.empty(), - created, - removed); - continue; - } - startBuild(plan.get().sourceFiles(), plan.get().inputUnits()); + Optional> plan = + levels.pick(groups, activeSourceFiles); + if (plan.isPresent()) { + if (plan.get().sourceFiles().isEmpty()) { + replaceInputGroups( + plan.get().inputUnits(), Optional.empty(), created, removed); + continue; } + startBuild(plan.get()); } } if (!waitCompaction || pendingBuild == null) { @@ -246,18 +236,6 @@ private void applySourceTransition(CompactIncrement compactIncrement) { } } - @Nullable - private DataFileMeta firstUncoveredSource() { - List candidates = new ArrayList<>(activeSourceFiles.values()); - candidates.sort(Comparator.comparing(DataFileMeta::fileName)); - for (DataFileMeta candidate : candidates) { - if (!isCovered(candidate, Collections.emptyList())) { - return candidate; - } - } - return null; - } - private boolean isCovered(DataFileMeta candidate, List excludedGroups) { for (PkSortedIndexGroup group : groups) { if (excludedGroups.contains(group)) { @@ -273,8 +251,8 @@ private boolean isCovered(DataFileMeta candidate, List exclu return false; } - private void startBuild(List sourceFiles, List inputGroups) { - PendingBuild next = new PendingBuild(sourceFiles, inputGroups); + private void startBuild(PrimaryKeyIndexLevels.Plan plan) { + PendingBuild next = new PendingBuild(plan); next.start(); pendingBuild = next; } @@ -287,8 +265,7 @@ private Optional finishPendingBuild(boolean blocking) throws Exc try { IndexFileMeta payload = completed.get(); pendingBuild = null; - return Optional.of( - new CompletedBuild(completed.sourceFiles, completed.inputGroups, payload)); + return Optional.of(new CompletedBuild(completed.plan, payload)); } catch (CancellationException e) { pendingBuild = null; throw e; @@ -307,6 +284,10 @@ private Optional finishPendingBuild(boolean blocking) throws Exc private void acceptOrDelete( CompletedBuild completed, List created, List removed) { + if (!levels.isCurrent(completed.plan, activeSourceFiles)) { + deleteGenerated(completed.payload); + return; + } List sources = new ArrayList<>(); boolean sourcesStillActive = true; for (DataFileMeta sourceFile : completed.sourceFiles) { @@ -325,7 +306,18 @@ private void acceptOrDelete( break; } } - if (!sourcesStillActive || !inputsStillPresent || outputOverlapsRetainedGroup) { + PrimaryKeyIndexSourceMeta outputSourceMeta; + try { + outputSourceMeta = PrimaryKeyIndexSourceMeta.fromIndexFile(completed.payload); + } catch (RuntimeException e) { + deleteGenerated(completed.payload); + return; + } + if (!sourcesStillActive + || !inputsStillPresent + || outputOverlapsRetainedGroup + || outputSourceMeta.dataLevel() != completed.plan.dataLevel() + || !outputSourceMeta.sourceFiles().equals(sources)) { deleteGenerated(completed.payload); return; } @@ -386,7 +378,6 @@ public synchronized boolean buildNotCompleted() { public synchronized boolean hasPendingMaintenance() { return pendingBuild != null || !pendingRestoredDeletions.isEmpty() - || firstUncoveredSource() != null || levels.pick(groups, activeSourceFiles).isPresent(); } @@ -408,16 +399,8 @@ public synchronized void close() { } public synchronized PkSortedBucketIndexState state() { - return PkSortedBucketIndexState.fromActivePayloads( - fieldId, indexType, sourceFiles(), activePayloads()); - } - - private List sourceFiles() { - List sources = new ArrayList<>(); - for (DataFileMeta dataFile : activeSourceFiles.values()) { - sources.add(new PrimaryKeyIndexSourceFile(dataFile.fileName(), dataFile.rowCount())); - } - return sources; + return PkSortedBucketIndexState.fromActiveDataFiles( + fieldId, indexType, new ArrayList<>(activeSourceFiles.values()), activePayloads()); } private List activePayloads() { @@ -430,15 +413,17 @@ private List activePayloads() { private final class PendingBuild { + private final PrimaryKeyIndexLevels.Plan plan; private final List sourceFiles; private final List inputGroups; @Nullable private IndexFileMeta result; @Nullable private Future future; private boolean cancelled; - private PendingBuild(List sourceFiles, List inputGroups) { - this.sourceFiles = Collections.unmodifiableList(new ArrayList<>(sourceFiles)); - this.inputGroups = Collections.unmodifiableList(new ArrayList<>(inputGroups)); + private PendingBuild(PrimaryKeyIndexLevels.Plan plan) { + this.plan = plan; + this.sourceFiles = plan.sourceFiles(); + this.inputGroups = plan.inputUnits(); } private void start() { @@ -506,16 +491,16 @@ private void cancel() { private static final class CompletedBuild { + private final PrimaryKeyIndexLevels.Plan plan; private final List sourceFiles; private final List inputGroups; private final IndexFileMeta payload; private CompletedBuild( - List sourceFiles, - List inputGroups, - IndexFileMeta payload) { - this.sourceFiles = sourceFiles; - this.inputGroups = inputGroups; + PrimaryKeyIndexLevels.Plan plan, IndexFileMeta payload) { + this.plan = plan; + this.sourceFiles = plan.sourceFiles(); + this.inputGroups = plan.inputUnits(); this.payload = payload; } } diff --git a/paimon-core/src/main/java/org/apache/paimon/index/pksorted/PkSortedBucketIndexState.java b/paimon-core/src/main/java/org/apache/paimon/index/pksorted/PkSortedBucketIndexState.java index 078c719680d6..fa6a5dd10b06 100644 --- a/paimon-core/src/main/java/org/apache/paimon/index/pksorted/PkSortedBucketIndexState.java +++ b/paimon-core/src/main/java/org/apache/paimon/index/pksorted/PkSortedBucketIndexState.java @@ -21,15 +21,18 @@ import org.apache.paimon.index.IndexFileMeta; import org.apache.paimon.index.pk.PrimaryKeyIndexSourceFile; import org.apache.paimon.index.pk.PrimaryKeyIndexSourceMeta; +import org.apache.paimon.index.pk.PrimaryKeyIndexSourcePolicy; +import org.apache.paimon.io.DataFileMeta; import java.util.ArrayList; import java.util.Collections; +import java.util.Comparator; import java.util.HashSet; -import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; +import java.util.TreeMap; /** Immutable sorted-index state for one field and bucket. */ public final class PkSortedBucketIndexState { @@ -50,56 +53,70 @@ private PkSortedBucketIndexState( this.rejectedPayloads = Collections.unmodifiableList(rejectedPayloads); } - public static PkSortedBucketIndexState fromActivePayloads( + public static PkSortedBucketIndexState fromActiveDataFiles( int fieldId, String indexType, - List activeSourceFiles, + List activeDataFiles, List activePayloads) { - Map, List> payloadsBySources = - new LinkedHashMap<>(); + Map> sourcesByLevel = new TreeMap<>(); + for (DataFileMeta dataFile : activeDataFiles) { + if (PrimaryKeyIndexSourcePolicy.shouldRead(dataFile)) { + sourcesByLevel + .computeIfAbsent(dataFile.level(), ignored -> new ArrayList<>()) + .add( + new PrimaryKeyIndexSourceFile( + dataFile.fileName(), dataFile.rowCount())); + } + } + for (List sources : sourcesByLevel.values()) { + sources.sort(Comparator.comparing(PrimaryKeyIndexSourceFile::fileName)); + } + + Map> payloadsByLevel = new TreeMap<>(); List rejected = new ArrayList<>(); for (IndexFileMeta payload : activePayloads) { try { - List sourceFiles = - PrimaryKeyIndexSourceMeta.fromIndexFile(payload).sourceFiles(); - payloadsBySources - .computeIfAbsent(sourceFiles, key -> new ArrayList<>()) - .add(payload); + PrimaryKeyIndexSourceMeta sourceMeta = + PrimaryKeyIndexSourceMeta.fromIndexFile(payload); + List desired = + sourcesByLevel.get(sourceMeta.dataLevel()); + if (desired == null || !desired.equals(sourceMeta.sourceFiles())) { + rejected.add(payload); + } else { + payloadsByLevel + .computeIfAbsent(sourceMeta.dataLevel(), ignored -> new ArrayList<>()) + .add(payload); + } } catch (RuntimeException ignored) { rejected.add(payload); } } List groups = new ArrayList<>(); - Set activeSet = new HashSet<>(activeSourceFiles); - Set coveredSet = new HashSet<>(); - for (Map.Entry, List> entry : - payloadsBySources.entrySet()) { + Set coveredLevels = new HashSet<>(); + for (Map.Entry> entry : payloadsByLevel.entrySet()) { + List levelPayloads = entry.getValue(); Optional group = - PkSortedIndexGroup.create(fieldId, indexType, entry.getKey(), entry.getValue()); - boolean overlapsActiveSource = false; - for (PrimaryKeyIndexSourceFile sourceFile : entry.getKey()) { - if (activeSet.contains(sourceFile) && coveredSet.contains(sourceFile)) { - overlapsActiveSource = true; - break; - } - } - if (group.isPresent() && !overlapsActiveSource) { + levelPayloads.size() == 1 + ? PkSortedIndexGroup.create( + fieldId, + indexType, + sourcesByLevel.get(entry.getKey()), + levelPayloads) + : Optional.empty(); + if (group.isPresent()) { groups.add(group.get()); - coveredSet.addAll(entry.getKey()); + coveredLevels.add(entry.getKey()); } else { - rejected.addAll(entry.getValue()); + rejected.addAll(levelPayloads); } } List covered = new ArrayList<>(); List uncovered = new ArrayList<>(); - for (PrimaryKeyIndexSourceFile sourceFile : activeSourceFiles) { - if (coveredSet.contains(sourceFile)) { - covered.add(sourceFile); - } else { - uncovered.add(sourceFile); - } + for (Map.Entry> entry : + sourcesByLevel.entrySet()) { + (coveredLevels.contains(entry.getKey()) ? covered : uncovered).addAll(entry.getValue()); } return new PkSortedBucketIndexState(groups, covered, uncovered, rejected); } diff --git a/paimon-core/src/main/java/org/apache/paimon/index/pksorted/PkSortedIndexBuilder.java b/paimon-core/src/main/java/org/apache/paimon/index/pksorted/PkSortedIndexBuilder.java index ff78d72a6aa2..ef3c210e2156 100644 --- a/paimon-core/src/main/java/org/apache/paimon/index/pksorted/PkSortedIndexBuilder.java +++ b/paimon-core/src/main/java/org/apache/paimon/index/pksorted/PkSortedIndexBuilder.java @@ -86,6 +86,15 @@ public IndexFileMeta build(List dataFiles) throws IOException { checkArgument(!dataFiles.isEmpty(), "A sorted index build requires source files."); List orderedDataFiles = new ArrayList<>(dataFiles); orderedDataFiles.sort(Comparator.comparing(DataFileMeta::fileName)); + int dataLevel = orderedDataFiles.get(0).level(); + checkArgument(dataLevel > 0, "A sorted index build requires a positive data level."); + for (DataFileMeta dataFile : orderedDataFiles) { + checkArgument( + dataFile.level() == dataLevel, + "A sorted index build cannot mix data levels %s and %s.", + dataLevel, + dataFile.level()); + } IOManager actualIOManager = ioManager; boolean ownsIOManager = false; @@ -163,7 +172,8 @@ public PkSortedIndexFile.Entry next() { valueGetter.getFieldOrNull(row), row.getLong(1)); } }; - return indexFile.build(sourceFiles, indexField, indexType, options, sortedEntries); + return indexFile.build( + dataLevel, sourceFiles, indexField, indexType, options, sortedEntries); } finally { if (sortBuffer != null) { sortBuffer.clear(); diff --git a/paimon-core/src/main/java/org/apache/paimon/index/pksorted/PkSortedIndexFile.java b/paimon-core/src/main/java/org/apache/paimon/index/pksorted/PkSortedIndexFile.java index 7737393d25f5..944cd4ef326c 100644 --- a/paimon-core/src/main/java/org/apache/paimon/index/pksorted/PkSortedIndexFile.java +++ b/paimon-core/src/main/java/org/apache/paimon/index/pksorted/PkSortedIndexFile.java @@ -54,6 +54,7 @@ public PkSortedIndexFile(FileIO fileIO, IndexPathFactory pathFactory) { } public IndexFileMeta build( + int dataLevel, List sourceFiles, DataField indexField, String indexType, @@ -101,7 +102,7 @@ public IndexFileMeta build( "Sorted payload row count %s does not match source row count %s.", result.rowCount(), sourceRowCount); - byte[] sourceMeta = new PrimaryKeyIndexSourceMeta(sourceFiles).serialize(); + byte[] sourceMeta = new PrimaryKeyIndexSourceMeta(dataLevel, sourceFiles).serialize(); Path payloadPath = fileWriter.path(result.fileName()); IndexFileMeta payload = new IndexFileMeta( diff --git a/paimon-core/src/main/java/org/apache/paimon/index/pksorted/PkSortedIndexGroup.java b/paimon-core/src/main/java/org/apache/paimon/index/pksorted/PkSortedIndexGroup.java index bd0acfe8c97c..54999c216247 100644 --- a/paimon-core/src/main/java/org/apache/paimon/index/pksorted/PkSortedIndexGroup.java +++ b/paimon-core/src/main/java/org/apache/paimon/index/pksorted/PkSortedIndexGroup.java @@ -30,30 +30,30 @@ import java.util.Optional; import java.util.Set; -/** All rotated payloads that index the same ordered source data files. */ +/** The single payload which indexes one complete data level. */ public final class PkSortedIndexGroup { + private final int dataLevel; private final List sourceFiles; private final List payloads; - PkSortedIndexGroup(List sourceFiles, List payloads) { + PkSortedIndexGroup( + int dataLevel, + List sourceFiles, + List payloads) { + this.dataLevel = dataLevel; this.sourceFiles = Collections.unmodifiableList(new ArrayList<>(sourceFiles)); this.payloads = Collections.unmodifiableList(new ArrayList<>(payloads)); } - static Optional create( - int fieldId, - String indexType, - PrimaryKeyIndexSourceFile sourceFile, - List payloads) { - return create(fieldId, indexType, Collections.singletonList(sourceFile), payloads); - } - static Optional create( int fieldId, String indexType, List sourceFiles, List payloads) { + if (payloads.size() != 1) { + return Optional.empty(); + } long sourceRowCount = 0; Set sourceNames = new HashSet<>(); for (PrimaryKeyIndexSourceFile sourceFile : sourceFiles) { @@ -72,12 +72,14 @@ static Optional create( long payloadRowCount = 0; Set payloadNames = new HashSet<>(); + Integer dataLevel = null; for (IndexFileMeta payload : payloads) { GlobalIndexMeta meta = payload.globalIndexMeta(); - List payloadSources = - PrimaryKeyIndexSourceMeta.fromIndexFile(payload).sourceFiles(); + PrimaryKeyIndexSourceMeta sourceMeta = PrimaryKeyIndexSourceMeta.fromIndexFile(payload); + List payloadSources = sourceMeta.sourceFiles(); if (!payloadNames.add(payload.fileName()) || !sourceFiles.equals(payloadSources) + || (dataLevel != null && dataLevel != sourceMeta.dataLevel()) || !indexType.equals(payload.indexType()) || meta == null || meta.indexFieldId() != fieldId @@ -85,25 +87,21 @@ static Optional create( || meta.rowRangeEnd() != sourceRowCount - 1) { return Optional.empty(); } + dataLevel = sourceMeta.dataLevel(); try { payloadRowCount = Math.addExact(payloadRowCount, payload.rowCount()); } catch (ArithmeticException e) { return Optional.empty(); } } - if (payloadRowCount != sourceRowCount) { + if (dataLevel == null || payloadRowCount != sourceRowCount) { return Optional.empty(); } - return Optional.of(new PkSortedIndexGroup(sourceFiles, payloads)); + return Optional.of(new PkSortedIndexGroup(dataLevel, sourceFiles, payloads)); } - public PrimaryKeyIndexSourceFile sourceFile() { - if (sourceFiles.size() != 1) { - throw new IllegalStateException( - String.format( - "Expected exactly one source file, but found %s.", sourceFiles.size())); - } - return sourceFiles.get(0); + public int dataLevel() { + return dataLevel; } public List sourceFiles() { @@ -113,17 +111,4 @@ public List sourceFiles() { public List payloads() { return payloads; } - - public String identity() { - List names = new ArrayList<>(); - for (IndexFileMeta payload : payloads) { - names.add(payload.fileName()); - } - Collections.sort(names); - StringBuilder identity = new StringBuilder(); - for (String name : names) { - identity.append(name.length()).append(':').append(name); - } - return identity.toString(); - } } diff --git a/paimon-core/src/main/java/org/apache/paimon/index/pkvector/BucketedVectorIndexMaintainer.java b/paimon-core/src/main/java/org/apache/paimon/index/pkvector/BucketedVectorIndexMaintainer.java index 014e7a3e34b4..b923e052b978 100644 --- a/paimon-core/src/main/java/org/apache/paimon/index/pkvector/BucketedVectorIndexMaintainer.java +++ b/paimon-core/src/main/java/org/apache/paimon/index/pkvector/BucketedVectorIndexMaintainer.java @@ -18,7 +18,6 @@ package org.apache.paimon.index.pkvector; -import org.apache.paimon.CoreOptions; import org.apache.paimon.data.BinaryRow; import org.apache.paimon.deletionvectors.DeletionVector; import org.apache.paimon.index.IndexFileHandler; @@ -41,7 +40,6 @@ import java.io.UncheckedIOException; import java.util.ArrayList; import java.util.Collections; -import java.util.Comparator; import java.util.HashSet; import java.util.LinkedHashMap; import java.util.List; @@ -70,6 +68,7 @@ public class BucketedVectorIndexMaintainer { private final PrimaryKeyIndexLevels annLevels; private ExecutorService executor; private final List annSegments; + private final List retiredSegments; private final Map activeSourceFiles; @Nullable private PendingBuild pendingBuild; @@ -118,16 +117,19 @@ public class BucketedVectorIndexMaintainer { this.algorithm = algorithm; this.vectorReaderFactory = vectorReaderFactory; this.deletionVectorFactory = deletionVectorFactory; - CoreOptions coreOptions = new CoreOptions(indexOptions); this.annLevels = new PrimaryKeyIndexLevels<>( - coreOptions.primaryKeyIndexCompactionLevelFanout(vectorField.name()), - coreOptions.primaryKeyIndexCompactionStaleRatioThreshold( - vectorField.name()), - IndexFileMeta::fileName, + segment -> sourceMeta(segment).dataLevel(), segment -> sourceMeta(segment).sourceFiles()); this.executor = executor; + this.activeSourceFiles = new LinkedHashMap<>(); + for (DataFileMeta file : restoredDataFiles) { + if (PrimaryKeyIndexSourcePolicy.shouldRead(file)) { + activeSourceFiles.put(file.fileName(), file); + } + } + List definitionPayloads = new ArrayList<>(); for (IndexFileMeta payload : restoredPayloads) { if (algorithm.equals(payload.indexType()) @@ -137,14 +139,13 @@ public class BucketedVectorIndexMaintainer { } } PkVectorBucketIndexState restoredState = - new PkVectorBucketIndexState(vectorFieldId, algorithm, definitionPayloads); + PkVectorBucketIndexState.fromActiveDataFiles( + vectorFieldId, + algorithm, + new ArrayList<>(activeSourceFiles.values()), + definitionPayloads); this.annSegments = new ArrayList<>(restoredState.annSegments()); - this.activeSourceFiles = new LinkedHashMap<>(); - for (DataFileMeta file : restoredDataFiles) { - if (PrimaryKeyIndexSourcePolicy.shouldRead(file)) { - activeSourceFiles.put(file.fileName(), file); - } - } + this.retiredSegments = new ArrayList<>(restoredState.staleSegments()); validateCoverage(annSegments, activeSourceFiles); } @@ -159,6 +160,7 @@ public synchronized VectorIndexCommit prepareCommit( "Append files must not be primary-key vector index sources."); List originalSegments = new ArrayList<>(annSegments); + List originalRetired = new ArrayList<>(retiredSegments); Map originalSourceFiles = new LinkedHashMap<>(activeSourceFiles); List generated = new ArrayList<>(); try { @@ -174,7 +176,8 @@ public synchronized VectorIndexCommit prepareCommit( } } - List removed = new ArrayList<>(); + List removed = new ArrayList<>(retiredSegments); + retiredSegments.clear(); List created = new ArrayList<>(); activeSourceFiles.clear(); @@ -189,23 +192,19 @@ public synchronized VectorIndexCommit prepareCommit( replaceSegments(build, created, removed); } else { annSegmentFile.delete(build.segment); + generated.remove(build.segment); } } if (pendingBuild == null) { - List uncovered = uncoveredFiles(); - if (!uncovered.isEmpty()) { - startPendingBuild(uncovered, Collections.emptyList()); - } else { - Optional> plan = - annLevels.pick(annSegments, activeSourceFiles); - if (plan.isPresent()) { - if (plan.get().sourceFiles().isEmpty()) { - removeSegments(plan.get().inputUnits(), created, removed); - continue; - } - startPendingBuild(plan.get().sourceFiles(), plan.get().inputUnits()); + Optional> plan = + annLevels.pick(annSegments, activeSourceFiles); + if (plan.isPresent()) { + if (plan.get().sourceFiles().isEmpty()) { + removeSegments(plan.get().inputUnits(), created, removed); + continue; } + startPendingBuild(plan.get()); } } if (!waitCompaction || pendingBuild == null) { @@ -231,9 +230,14 @@ public synchronized VectorIndexCommit prepareCommit( compactChange, failure -> rollbackPrepareCommit( - originalSegments, originalSourceFiles, generated, failure)); + originalSegments, + originalRetired, + originalSourceFiles, + generated, + failure)); } catch (Throwable failure) { - rollbackPrepareCommit(originalSegments, originalSourceFiles, generated, failure); + rollbackPrepareCommit( + originalSegments, originalRetired, originalSourceFiles, generated, failure); if (failure instanceof Exception) { throw (Exception) failure; } @@ -244,20 +248,23 @@ public synchronized VectorIndexCommit prepareCommit( } } - private void startPendingBuild( - List sourceFiles, List inputSegments) throws IOException { - PendingBuild build = new PendingBuild(sourceFiles, inputSegments); + private void startPendingBuild(PrimaryKeyIndexLevels.Plan plan) + throws IOException { + PendingBuild build = new PendingBuild(plan); build.start(); pendingBuild = build; } private synchronized void rollbackPrepareCommit( List originalSegments, + List originalRetired, Map originalSourceFiles, List generated, Throwable failure) { annSegments.clear(); annSegments.addAll(originalSegments); + retiredSegments.clear(); + retiredSegments.addAll(originalRetired); activeSourceFiles.clear(); activeSourceFiles.putAll(originalSourceFiles); @@ -300,18 +307,6 @@ private void removeSegments( } } - private List uncoveredFiles() { - Set covered = coveredSources(annSegments); - List uncovered = new ArrayList<>(); - for (DataFileMeta file : activeSourceFiles.values()) { - if (!covered.contains(file.fileName())) { - uncovered.add(file); - } - } - uncovered.sort(Comparator.comparing(DataFileMeta::fileName)); - return uncovered; - } - private Optional finishPendingBuild(boolean blocking) throws Exception { if (pendingBuild == null || (!blocking && !pendingBuild.isDone())) { return Optional.empty(); @@ -321,7 +316,7 @@ private Optional finishPendingBuild(boolean blocking) throws Exc try { IndexFileMeta segment = completed.get(); pendingBuild = null; - return Optional.of(new CompletedBuild(segment, completed.inputSegments)); + return Optional.of(new CompletedBuild(completed.plan, segment)); } catch (CancellationException e) { pendingBuild = null; return Optional.empty(); @@ -339,15 +334,30 @@ private Optional finishPendingBuild(boolean blocking) throws Exc } private boolean canAccept(CompletedBuild build) { - if (!annSegments.containsAll(build.inputSegments)) { + if (!annLevels.isCurrent(build.plan, activeSourceFiles) + || !annSegments.containsAll(build.inputSegments)) { + return false; + } + PrimaryKeyIndexSourceMeta outputSourceMeta; + try { + outputSourceMeta = sourceMeta(build.segment); + } catch (RuntimeException e) { + return false; + } + if (outputSourceMeta.dataLevel() != build.plan.dataLevel() + || outputSourceMeta.sourceFiles().size() != build.plan.sourceFiles().size()) { return false; } List retained = new ArrayList<>(annSegments); retained.removeAll(build.inputSegments); Set covered = coveredSources(retained); - for (PrimaryKeyIndexSourceFile source : sourceMeta(build.segment).sourceFiles()) { + for (int i = 0; i < outputSourceMeta.sourceFiles().size(); i++) { + PrimaryKeyIndexSourceFile source = outputSourceMeta.sourceFiles().get(i); + DataFileMeta expected = build.plan.sourceFiles().get(i); DataFileMeta activeFile = activeSourceFiles.get(source.fileName()); - if (activeFile == null + if (!source.fileName().equals(expected.fileName()) + || source.rowCount() != expected.rowCount() + || activeFile == null || activeFile.rowCount() != source.rowCount() || covered.contains(source.fileName())) { return false; @@ -385,6 +395,7 @@ private IndexFileMeta buildAnnSegment( deletionVector != null ? deletionVector::isDeleted : position -> false; sources.add( PkVectorAnnSegmentFile.Source.lazy( + file.level(), sourceFile, () -> vectorReaderFactory.create(file), excludedPosition)); @@ -413,7 +424,15 @@ private Map snapshotDeletionVectors(List f private void validateCoverage( List candidateAnn, Map sourceFiles) { PkVectorBucketIndexState state = - new PkVectorBucketIndexState(vectorFieldId, algorithm, candidateAnn); + PkVectorBucketIndexState.fromActiveDataFiles( + vectorFieldId, + algorithm, + new ArrayList<>(sourceFiles.values()), + candidateAnn); + checkArgument( + state.staleSegments().isEmpty() + && state.annSegments().size() == candidateAnn.size(), + "ANN segments must each exactly cover one current data level."); for (Map.Entry entry : state.sourceFileToAnnSegment().entrySet()) { DataFileMeta file = sourceFiles.get(entry.getKey()); if (file == null) { @@ -436,11 +455,13 @@ public synchronized List segments() { } public synchronized PkVectorBucketIndexState state() { - return new PkVectorBucketIndexState(vectorFieldId, algorithm, annSegments); + return PkVectorBucketIndexState.fromActiveDataFiles( + vectorFieldId, algorithm, new ArrayList<>(activeSourceFiles.values()), annSegments); } private class PendingBuild { + private final PrimaryKeyIndexLevels.Plan plan; private final List sourceFiles; private final List inputSegments; private final Map deletionVectors; @@ -448,10 +469,10 @@ private class PendingBuild { @Nullable private Future future; private boolean cancelled; - private PendingBuild(List sourceFiles, List inputSegments) - throws IOException { - this.sourceFiles = new ArrayList<>(sourceFiles); - this.inputSegments = new ArrayList<>(inputSegments); + private PendingBuild(PrimaryKeyIndexLevels.Plan plan) throws IOException { + this.plan = plan; + this.sourceFiles = plan.sourceFiles(); + this.inputSegments = plan.inputUnits(); this.deletionVectors = snapshotDeletionVectors(sourceFiles); } @@ -500,12 +521,15 @@ private void cancel() { private static class CompletedBuild { + private final PrimaryKeyIndexLevels.Plan plan; private final IndexFileMeta segment; private final List inputSegments; - private CompletedBuild(IndexFileMeta segment, List inputSegments) { + private CompletedBuild( + PrimaryKeyIndexLevels.Plan plan, IndexFileMeta segment) { + this.plan = plan; this.segment = segment; - this.inputSegments = new ArrayList<>(inputSegments); + this.inputSegments = plan.inputUnits(); } } diff --git a/paimon-core/src/main/java/org/apache/paimon/index/pkvector/PkVectorAnnSegmentFile.java b/paimon-core/src/main/java/org/apache/paimon/index/pkvector/PkVectorAnnSegmentFile.java index c466024f699a..e337db2802d2 100644 --- a/paimon-core/src/main/java/org/apache/paimon/index/pkvector/PkVectorAnnSegmentFile.java +++ b/paimon-core/src/main/java/org/apache/paimon/index/pkvector/PkVectorAnnSegmentFile.java @@ -67,9 +67,16 @@ public IndexFileMeta build( String indexType) throws IOException { checkArgument(!sources.isEmpty(), "An ANN segment must reference source files."); + int dataLevel = sources.get(0).dataLevel; + checkArgument(dataLevel > 0, "An ANN segment requires a positive data level."); long totalRowCount = 0; List sourceFiles = new ArrayList<>(sources.size()); for (Source source : sources) { + checkArgument( + source.dataLevel == dataLevel, + "An ANN segment cannot mix data levels %s and %s.", + dataLevel, + source.dataLevel); totalRowCount = Math.addExact(totalRowCount, source.sourceFile.rowCount()); sourceFiles.add(source.sourceFile); } @@ -165,7 +172,8 @@ public IndexFileMeta build( vectorField.id(), null, payloadMetadata, - new PrimaryKeyIndexSourceMeta(sourceFiles).serialize()), + new PrimaryKeyIndexSourceMeta(dataLevel, sourceFiles) + .serialize()), pathFactory.isExternalPath() ? payloadPath.toString() : null); success = true; return segment; @@ -229,6 +237,7 @@ private void deleteCreatedFiles() { /** One vector source used while building an ANN segment. */ public static class Source { + private final int dataLevel; private final PrimaryKeyIndexSourceFile sourceFile; @Nullable private final PkVectorReader vectors; @Nullable private final ReaderFactory readerFactory; @@ -240,13 +249,15 @@ public Source(DataFileMeta sourceFile, PkVectorReader vectors) { public Source( DataFileMeta sourceFile, PkVectorReader vectors, LongPredicate excludedPosition) { - this(sourceMetadata(sourceFile), vectors, excludedPosition); + this(sourceFile.level(), sourceMetadata(sourceFile), vectors, excludedPosition); } Source( + int dataLevel, PrimaryKeyIndexSourceFile sourceFile, PkVectorReader vectors, LongPredicate excludedPosition) { + this.dataLevel = dataLevel; this.sourceFile = sourceFile; this.vectors = vectors; this.readerFactory = null; @@ -254,24 +265,28 @@ public Source( } private Source( + int dataLevel, PrimaryKeyIndexSourceFile sourceFile, ReaderFactory readerFactory, LongPredicate excludedPosition) { + this.dataLevel = dataLevel; this.sourceFile = sourceFile; this.vectors = null; this.readerFactory = readerFactory; this.excludedPosition = excludedPosition; } - static Source lazy(PrimaryKeyIndexSourceFile sourceFile, ReaderFactory readerFactory) { - return new Source(sourceFile, readerFactory, position -> false); + static Source lazy( + int dataLevel, PrimaryKeyIndexSourceFile sourceFile, ReaderFactory readerFactory) { + return new Source(dataLevel, sourceFile, readerFactory, position -> false); } static Source lazy( + int dataLevel, PrimaryKeyIndexSourceFile sourceFile, ReaderFactory readerFactory, LongPredicate excludedPosition) { - return new Source(sourceFile, readerFactory, excludedPosition); + return new Source(dataLevel, sourceFile, readerFactory, excludedPosition); } private PkVectorReader openReader() throws IOException { diff --git a/paimon-core/src/main/java/org/apache/paimon/index/pkvector/PkVectorBucketIndexState.java b/paimon-core/src/main/java/org/apache/paimon/index/pkvector/PkVectorBucketIndexState.java index 313f59e73296..6fb6fe8a3af1 100644 --- a/paimon-core/src/main/java/org/apache/paimon/index/pkvector/PkVectorBucketIndexState.java +++ b/paimon-core/src/main/java/org/apache/paimon/index/pkvector/PkVectorBucketIndexState.java @@ -21,11 +21,16 @@ import org.apache.paimon.index.IndexFileMeta; import org.apache.paimon.index.pk.PrimaryKeyIndexSourceFile; import org.apache.paimon.index.pk.PrimaryKeyIndexSourceMeta; +import org.apache.paimon.index.pk.PrimaryKeyIndexSourcePolicy; +import org.apache.paimon.io.DataFileMeta; +import java.util.ArrayList; import java.util.Collections; +import java.util.Comparator; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.TreeMap; import static org.apache.paimon.utils.Preconditions.checkArgument; @@ -35,11 +40,65 @@ public final class PkVectorBucketIndexState { private final int vectorFieldId; private final String indexType; private final List annSegments; + private final List staleSegments; private final Map sourceFileToAnnSegment; - public static PkVectorBucketIndexState fromActivePayloads( - int vectorFieldId, String indexType, List activePayloads) { - return new PkVectorBucketIndexState(vectorFieldId, indexType, activePayloads); + public static PkVectorBucketIndexState fromActiveDataFiles( + int vectorFieldId, + String indexType, + List activeDataFiles, + List activePayloads) { + Map> sourcesByLevel = new TreeMap<>(); + for (DataFileMeta dataFile : activeDataFiles) { + if (PrimaryKeyIndexSourcePolicy.shouldRead(dataFile)) { + sourcesByLevel + .computeIfAbsent(dataFile.level(), ignored -> new ArrayList<>()) + .add( + new PrimaryKeyIndexSourceFile( + dataFile.fileName(), dataFile.rowCount())); + } + } + for (List sources : sourcesByLevel.values()) { + sources.sort(Comparator.comparing(PrimaryKeyIndexSourceFile::fileName)); + } + + Map> segmentsByLevel = new TreeMap<>(); + List stale = new ArrayList<>(); + for (IndexFileMeta segment : activePayloads) { + try { + PkVectorBucketIndexState singleton = + new PkVectorBucketIndexState( + vectorFieldId, indexType, Collections.singletonList(segment)); + PrimaryKeyIndexSourceMeta sourceMeta = + PrimaryKeyIndexSourceMeta.fromIndexFile(segment); + List desired = + sourcesByLevel.get(sourceMeta.dataLevel()); + if (singleton.annSegments().size() != 1 + || desired == null + || !desired.equals(sourceMeta.sourceFiles())) { + stale.add(segment); + } else { + segmentsByLevel + .computeIfAbsent(sourceMeta.dataLevel(), ignored -> new ArrayList<>()) + .add(segment); + } + } catch (RuntimeException ignored) { + stale.add(segment); + } + } + + List current = new ArrayList<>(); + for (List levelSegments : segmentsByLevel.values()) { + if (levelSegments.size() == 1) { + current.add(levelSegments.get(0)); + } else { + stale.addAll(levelSegments); + } + } + PkVectorBucketIndexState state = + new PkVectorBucketIndexState(vectorFieldId, indexType, current); + return new PkVectorBucketIndexState( + vectorFieldId, indexType, state.annSegments, stale, state.sourceFileToAnnSegment); } public PkVectorBucketIndexState( @@ -67,9 +126,24 @@ public PkVectorBucketIndexState( } this.annSegments = Collections.unmodifiableList(new java.util.ArrayList<>(activePayloads)); + this.staleSegments = Collections.emptyList(); this.sourceFileToAnnSegment = Collections.unmodifiableMap(annBySource); } + private PkVectorBucketIndexState( + int vectorFieldId, + String indexType, + List annSegments, + List staleSegments, + Map sourceFileToAnnSegment) { + this.vectorFieldId = vectorFieldId; + this.indexType = indexType; + this.annSegments = Collections.unmodifiableList(new ArrayList<>(annSegments)); + this.staleSegments = Collections.unmodifiableList(new ArrayList<>(staleSegments)); + this.sourceFileToAnnSegment = + Collections.unmodifiableMap(new LinkedHashMap<>(sourceFileToAnnSegment)); + } + public int vectorFieldId() { return vectorFieldId; } @@ -82,6 +156,10 @@ public List annSegments() { return annSegments; } + public List staleSegments() { + return staleSegments; + } + public Map sourceFileToAnnSegment() { return sourceFileToAnnSegment; } diff --git a/paimon-core/src/main/java/org/apache/paimon/schema/SchemaValidation.java b/paimon-core/src/main/java/org/apache/paimon/schema/SchemaValidation.java index 9f7431073680..7c743f416ec6 100644 --- a/paimon-core/src/main/java/org/apache/paimon/schema/SchemaValidation.java +++ b/paimon-core/src/main/java/org/apache/paimon/schema/SchemaValidation.java @@ -1024,24 +1024,6 @@ private static void validatePrimaryKeyIndexColumns(CoreOptions options) { validateUniquePrimaryKeyIndexColumns(indexedColumns, btreeColumns); validateUniquePrimaryKeyIndexColumns(indexedColumns, bitmapColumns); validateUniquePrimaryKeyIndexColumns(indexedColumns, fullTextColumns); - - Set compactedIndexColumns = new HashSet<>(); - compactedIndexColumns.addAll(vectorColumns); - compactedIndexColumns.addAll(btreeColumns); - compactedIndexColumns.addAll(bitmapColumns); - compactedIndexColumns.addAll(fullTextColumns); - for (String column : compactedIndexColumns) { - String fanoutKey = CoreOptions.primaryKeyIndexCompactionLevelFanoutKey(column); - checkArgument( - options.primaryKeyIndexCompactionLevelFanout(column) > 1, - "%s must be greater than 1.", - fanoutKey); - String staleRatioKey = - CoreOptions.primaryKeyIndexCompactionStaleRatioThresholdKey(column); - double staleRatio = options.primaryKeyIndexCompactionStaleRatioThreshold(column); - checkArgument( - staleRatio > 0 && staleRatio <= 1, "%s must be in (0, 1].", staleRatioKey); - } } private static void validateNoDuplicatePrimaryKeyIndexColumns( diff --git a/paimon-core/src/main/java/org/apache/paimon/table/source/PrimaryKeyFullTextScan.java b/paimon-core/src/main/java/org/apache/paimon/table/source/PrimaryKeyFullTextScan.java index f4b91a3e68c7..5b2799cf79ba 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/source/PrimaryKeyFullTextScan.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/source/PrimaryKeyFullTextScan.java @@ -193,8 +193,9 @@ static Plan plan( continue; } PkFullTextBucketIndexState state = - PkFullTextBucketIndexState.fromActivePayloads( + PkFullTextBucketIndexState.fromActiveDataFiles( textFieldId, + bucket.dataFiles(), payloads.getOrDefault(entry.getKey(), Collections.emptyList())); Set activeSources = bucket.dataFileNames(); Map currentPayloads = new LinkedHashMap<>(); diff --git a/paimon-core/src/main/java/org/apache/paimon/table/source/PrimaryKeySortedIndexScan.java b/paimon-core/src/main/java/org/apache/paimon/table/source/PrimaryKeySortedIndexScan.java index bcc98207b018..adea4965ac6d 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/source/PrimaryKeySortedIndexScan.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/source/PrimaryKeySortedIndexScan.java @@ -137,8 +137,7 @@ static Plan plan( } } - Map, List> sourcesByBucket = - new LinkedHashMap<>(); + Map, List> dataFilesByBucket = new LinkedHashMap<>(); for (DataSplit split : dataSplits) { checkArgument( split.snapshotId() == snapshotId, @@ -151,25 +150,25 @@ static Plan plan( checkArgument( deletions == null || deletions.size() == split.dataFiles().size(), "Deletion files must align with data files in a sorted-index split."); - List sources = - sourcesByBucket.computeIfAbsent( + List dataFiles = + dataFilesByBucket.computeIfAbsent( Pair.of(split.partition(), split.bucket()), ignored -> new ArrayList<>()); - for (DataFileMeta dataFile : split.dataFiles()) { - sources.add( - new PrimaryKeyIndexSourceFile(dataFile.fileName(), dataFile.rowCount())); - } + dataFiles.addAll(split.dataFiles()); } Map, Map>> groupsByBucket = new LinkedHashMap<>(); - for (Map.Entry, List> bucketEntry : - sourcesByBucket.entrySet()) { + for (Map.Entry, List> bucketEntry : + dataFilesByBucket.entrySet()) { Pair bucket = bucketEntry.getKey(); List bucketPayloads = payloadsByBucket.getOrDefault(bucket, Collections.emptyList()); - Set activeSourceFiles = - new HashSet<>(bucketEntry.getValue()); + Set activeSourceFiles = new HashSet<>(); + for (DataFileMeta dataFile : bucketEntry.getValue()) { + activeSourceFiles.add( + new PrimaryKeyIndexSourceFile(dataFile.fileName(), dataFile.rowCount())); + } Map> groupsBySource = new LinkedHashMap<>(); for (PrimaryKeyIndexDefinition definition : scalarDefinitions) { List definitionPayloads = new ArrayList<>(); @@ -183,7 +182,7 @@ static Plan plan( } try { PkSortedBucketIndexState state = - PkSortedBucketIndexState.fromActivePayloads( + PkSortedBucketIndexState.fromActiveDataFiles( definition.fieldId(), definition.indexType(), bucketEntry.getValue(), diff --git a/paimon-core/src/main/java/org/apache/paimon/table/source/PrimaryKeyVectorRead.java b/paimon-core/src/main/java/org/apache/paimon/table/source/PrimaryKeyVectorRead.java index 4a33127a294f..46232947f3b1 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/source/PrimaryKeyVectorRead.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/source/PrimaryKeyVectorRead.java @@ -288,8 +288,8 @@ CompletableFuture> searchBatchAsync( .filter(PrimaryKeyIndexSourcePolicy::shouldRead) .collect(Collectors.toList()); PkVectorBucketIndexState state = - PkVectorBucketIndexState.fromActivePayloads( - vectorField.id(), indexType, split.payloadFiles()); + PkVectorBucketIndexState.fromActiveDataFiles( + vectorField.id(), indexType, activeFiles, split.payloadFiles()); Map deletionVectors = deletionVectors(dataSplit, context.fileIO); PkVectorDataFileReader.Factory readerFactory = new PkVectorDataFileReader.Factory( diff --git a/paimon-core/src/test/java/org/apache/paimon/index/pk/BucketedPrimaryKeyIndexMaintainerTest.java b/paimon-core/src/test/java/org/apache/paimon/index/pk/BucketedPrimaryKeyIndexMaintainerTest.java index cb0955237a06..531f440930c8 100644 --- a/paimon-core/src/test/java/org/apache/paimon/index/pk/BucketedPrimaryKeyIndexMaintainerTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/index/pk/BucketedPrimaryKeyIndexMaintainerTest.java @@ -129,8 +129,6 @@ void testDelegatesFullTextLifecycleAndMergesCommit() throws Exception { void testFactoryCreatesConfiguredFullTextMaintainer() { Map options = new HashMap<>(); options.put(CoreOptions.PK_FULL_TEXT_INDEX_COLUMNS.key(), "content"); - options.put("fields.content.pk-index.compaction.level-fanout", "2"); - options.put("fields.content.pk-index.compaction.stale-ratio-threshold", "1.0"); TableSchema schema = new TableSchema( 0, @@ -321,8 +319,6 @@ void testNonBlockingCoordinatorStartsCoveredFanoutMaintenance() throws Exception release.await(); return merged; }, - 2, - 1.0, Arrays.asList(sourceA, sourceB), Arrays.asList(payloadA, payloadB), buildExecutor); @@ -413,8 +409,6 @@ private BucketedSortedIndexMaintainer sortedMaintainer( indexType, new PkSortedIndexFile(LocalFileIO.create(), pathFactory()), buildFunction, - 5, - 0.2, Collections.emptyList(), Collections.emptyList(), buildExecutor); @@ -475,13 +469,14 @@ private static IndexFileMeta payload( fieldId, null, new byte[] {1}, - new PrimaryKeyIndexSourceMeta(sources).serialize()), + new PrimaryKeyIndexSourceMeta(1, sources).serialize()), null); } private static IndexFileMeta fullTextPayload(String fileName, DataFileMeta sourceFile) { PrimaryKeyIndexSourceMeta sourceMeta = new PrimaryKeyIndexSourceMeta( + 1, new PrimaryKeyIndexSourceFile( sourceFile.fileName(), sourceFile.rowCount())); return new IndexFileMeta( diff --git a/paimon-core/src/test/java/org/apache/paimon/index/pk/PrimaryKeyIndexDefinitionsTest.java b/paimon-core/src/test/java/org/apache/paimon/index/pk/PrimaryKeyIndexDefinitionsTest.java index 378c1ad274e9..882c714e506b 100644 --- a/paimon-core/src/test/java/org/apache/paimon/index/pk/PrimaryKeyIndexDefinitionsTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/index/pk/PrimaryKeyIndexDefinitionsTest.java @@ -63,26 +63,6 @@ void testCreatesMixedDefinitionsInSchemaFieldOrder() { assertThat(definitions) .extracting(PrimaryKeyIndexDefinition::indexType) .containsExactly("btree", "bitmap", "ivf-pq"); - assertThat(definitions) - .extracting(PrimaryKeyIndexDefinition::compactionLevelFanout) - .containsOnly(5); - assertThat(definitions) - .extracting(PrimaryKeyIndexDefinition::compactionStaleRatioThreshold) - .containsOnly(0.2); - } - - @Test - void testResolvesFieldScopedCompactionOptions() { - Map options = new HashMap<>(); - options.put(CoreOptions.PK_BTREE_INDEX_COLUMNS.key(), "name"); - options.put("fields.name.pk-index.compaction.level-fanout", "7"); - options.put("fields.name.pk-index.compaction.stale-ratio-threshold", "0.4"); - - PrimaryKeyIndexDefinition definition = - PrimaryKeyIndexDefinitions.create(schema(options)).definitions().get(0); - - assertThat(definition.compactionLevelFanout()).isEqualTo(7); - assertThat(definition.compactionStaleRatioThreshold()).isEqualTo(0.4); } @Test @@ -117,21 +97,6 @@ void testResolvesFullTextIndexOptions() { .doesNotContainKey("fields.name.pk-full-text.index.options"); } - @Test - void testLegacyVectorCompactionOptionsAreIgnored() { - Map options = new HashMap<>(); - options.put(CoreOptions.PK_VECTOR_INDEX_COLUMNS.key(), "embedding"); - options.put("fields.embedding.pk-vector.index.type", "ivf-pq"); - options.put("pk-vector.index.compaction.level-fanout", "9"); - options.put("pk-vector.index.compaction.stale-ratio-threshold", "0.8"); - - PrimaryKeyIndexDefinition definition = - PrimaryKeyIndexDefinitions.create(schema(options)).definitions().get(0); - - assertThat(definition.compactionLevelFanout()).isEqualTo(5); - assertThat(definition.compactionStaleRatioThreshold()).isEqualTo(0.2); - } - @Test void testRejectsDuplicateColumnWithinFamily() { Map options = new HashMap<>(); diff --git a/paimon-core/src/test/java/org/apache/paimon/index/pk/PrimaryKeyIndexLevelsTest.java b/paimon-core/src/test/java/org/apache/paimon/index/pk/PrimaryKeyIndexLevelsTest.java index 989bb3857cb6..43977b8451ce 100644 --- a/paimon-core/src/test/java/org/apache/paimon/index/pk/PrimaryKeyIndexLevelsTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/index/pk/PrimaryKeyIndexLevelsTest.java @@ -31,91 +31,94 @@ import java.util.Map; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; /** Tests for {@link PrimaryKeyIndexLevels}. */ class PrimaryKeyIndexLevelsTest { @Test - void testPicksSimilarLogicalUnitsAtFanout() { + void testPlansCompleteMissingDataLevel() { PrimaryKeyIndexLevels levels = - new PrimaryKeyIndexLevels<>(3, 0.2, TestUnit::id, TestUnit::sources); - DataFileMeta dataA = dataFile("data-a", 30); - DataFileMeta dataB = dataFile("data-b", 40); - DataFileMeta dataC = dataFile("data-c", 50); - TestUnit unitA = unit("unit-a", dataA); - TestUnit unitB = unit("unit-b", dataB); - TestUnit unitC = unit("unit-c", dataC); - Map active = active(dataA, dataB, dataC); + new PrimaryKeyIndexLevels<>(TestUnit::dataLevel, TestUnit::sources); + DataFileMeta dataB = dataFile("data-b", 20, 2); + DataFileMeta dataA = dataFile("data-a", 10, 2); PrimaryKeyIndexLevels.Plan plan = - levels.pick(Arrays.asList(unitC, unitA, unitB), active).get(); + levels.pick(Collections.emptyList(), active(dataB, dataA)).get(); - assertThat(plan.inputUnits()).containsExactly(unitA, unitB, unitC); - assertThat(plan.sourceFiles()).containsExactly(dataA, dataB, dataC); + assertThat(plan.dataLevel()).isEqualTo(2); + assertThat(plan.inputUnits()).isEmpty(); + assertThat(plan.sourceFiles()).containsExactly(dataA, dataB); } @Test - void testPicksUnitAtStaleRatioThreshold() { + void testValidatesPlanAgainstCurrentCompleteLevel() { PrimaryKeyIndexLevels levels = - new PrimaryKeyIndexLevels<>(5, 0.4, TestUnit::id, TestUnit::sources); - DataFileMeta retired = dataFile("retired", 40); - DataFileMeta activeData = dataFile("active", 60); - TestUnit unit = unit("unit", retired, activeData); - + new PrimaryKeyIndexLevels<>(TestUnit::dataLevel, TestUnit::sources); + DataFileMeta dataA = dataFile("data-a", 10, 2); PrimaryKeyIndexLevels.Plan plan = - levels.pick(Collections.singletonList(unit), active(activeData)).get(); + levels.pick(Collections.emptyList(), active(dataA)).get(); - assertThat(plan.inputUnits()).containsExactly(unit); - assertThat(plan.sourceFiles()).containsExactly(activeData); + assertThat(levels.isCurrent(plan, active(dataA))).isTrue(); + assertThat(levels.isCurrent(plan, active(dataA, dataFile("data-b", 20, 2)))).isFalse(); } @Test - void testPicksUnitWithHighestStaleRatio() { + void testRebuildsMismatchedCompleteLevel() { PrimaryKeyIndexLevels levels = - new PrimaryKeyIndexLevels<>(5, 0.2, TestUnit::id, TestUnit::sources); - DataFileMeta activeA = dataFile("active-a", 50); - DataFileMeta activeB = dataFile("active-b", 20); - TestUnit halfStale = unit("unit-a", dataFile("retired-a", 50), activeA); - TestUnit mostlyStale = unit("unit-b", dataFile("retired-b", 80), activeB); + new PrimaryKeyIndexLevels<>(TestUnit::dataLevel, TestUnit::sources); + DataFileMeta dataA = dataFile("data-a", 30, 2); + DataFileMeta dataB = dataFile("data-b", 40, 2); + TestUnit partial = unit("partial", 2, dataA); PrimaryKeyIndexLevels.Plan plan = - levels.pick(Arrays.asList(halfStale, mostlyStale), active(activeA, activeB)).get(); + levels.pick(Collections.singletonList(partial), active(dataA, dataB)).get(); - assertThat(plan.inputUnits()).containsExactly(mostlyStale); + assertThat(plan.dataLevel()).isEqualTo(2); + assertThat(plan.inputUnits()).containsExactly(partial); + assertThat(plan.sourceFiles()).containsExactly(dataA, dataB); } @Test - void testBreaksEqualStaleRatioByIdentity() { + void testDropsLevelWithoutActiveData() { PrimaryKeyIndexLevels levels = - new PrimaryKeyIndexLevels<>(5, 0.2, TestUnit::id, TestUnit::sources); - TestUnit unitA = unit("unit-a", dataFile("retired-a", 10)); - TestUnit unitB = unit("unit-b", dataFile("retired-b", 10)); + new PrimaryKeyIndexLevels<>(TestUnit::dataLevel, TestUnit::sources); + TestUnit retired = unit("retired", 3, dataFile("data-a", 40, 3)); PrimaryKeyIndexLevels.Plan plan = - levels.pick(Arrays.asList(unitB, unitA), Collections.emptyMap()).get(); + levels.pick(Collections.singletonList(retired), Collections.emptyMap()).get(); - assertThat(plan.inputUnits()).containsExactly(unitA); + assertThat(plan.dataLevel()).isEqualTo(3); + assertThat(plan.inputUnits()).containsExactly(retired); assertThat(plan.sourceFiles()).isEmpty(); } @Test - void testSaturatesFanoutSizeComparison() { + void testExactLevelsNeedNoWork() { PrimaryKeyIndexLevels levels = - new PrimaryKeyIndexLevels<>(2, 1.0, TestUnit::id, TestUnit::sources); - DataFileMeta smaller = dataFile("data-a", Long.MAX_VALUE / 2 + 1); - DataFileMeta larger = dataFile("data-b", Long.MAX_VALUE - 1); - TestUnit unitA = unit("unit-a", smaller); - TestUnit unitB = unit("unit-b", larger); + new PrimaryKeyIndexLevels<>(TestUnit::dataLevel, TestUnit::sources); + DataFileMeta dataA = dataFile("data-a", 50, 2); + TestUnit current = unit("current", 2, dataA); - PrimaryKeyIndexLevels.Plan plan = - levels.pick(Arrays.asList(unitB, unitA), active(smaller, larger)).get(); + assertThat(levels.pick(Collections.singletonList(current), active(dataA))).isEmpty(); + } + + @Test + void testRejectsDuplicateUnitsForOneDataLevel() { + PrimaryKeyIndexLevels levels = + new PrimaryKeyIndexLevels<>(TestUnit::dataLevel, TestUnit::sources); + DataFileMeta dataA = dataFile("data-a", 10, 2); + TestUnit unitA = unit("unit-a", 2, dataA); + TestUnit unitB = unit("unit-b", 2, dataA); - assertThat(plan.inputUnits()).containsExactly(unitA, unitB); + assertThatThrownBy(() -> levels.pick(Arrays.asList(unitA, unitB), active(dataA))) + .hasMessageContaining("data level 2"); } - private static TestUnit unit(String id, DataFileMeta... files) { + private static TestUnit unit(String id, int dataLevel, DataFileMeta... files) { return new TestUnit( id, + dataLevel, Arrays.stream(files) .map( file -> @@ -133,35 +136,42 @@ private static Map active(DataFileMeta... files) { } private static DataFileMeta dataFile(String fileName, long rowCount) { + return dataFile(fileName, rowCount, 1); + } + + private static DataFileMeta dataFile(String fileName, long rowCount, int level) { return DataFileMeta.forAppend( - fileName, - 100, - rowCount, - SimpleStats.EMPTY_STATS, - 0, - 0, - 1, - Collections.emptyList(), - null, - FileSource.COMPACT, - null, - null, - null, - null); + fileName, + 100, + rowCount, + SimpleStats.EMPTY_STATS, + 0, + 0, + 1, + Collections.emptyList(), + null, + FileSource.COMPACT, + null, + null, + null, + null) + .upgrade(level); } private static final class TestUnit { private final String id; + private final int dataLevel; private final List sources; - private TestUnit(String id, List sources) { + private TestUnit(String id, int dataLevel, List sources) { this.id = id; + this.dataLevel = dataLevel; this.sources = sources; } - private String id() { - return id; + private int dataLevel() { + return dataLevel; } private List sources() { diff --git a/paimon-core/src/test/java/org/apache/paimon/index/pk/PrimaryKeyIndexSourceMetaTest.java b/paimon-core/src/test/java/org/apache/paimon/index/pk/PrimaryKeyIndexSourceMetaTest.java index e3bdab3eba38..8feb36077cb9 100644 --- a/paimon-core/src/test/java/org/apache/paimon/index/pk/PrimaryKeyIndexSourceMetaTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/index/pk/PrimaryKeyIndexSourceMetaTest.java @@ -20,6 +20,7 @@ import org.apache.paimon.index.GlobalIndexMeta; import org.apache.paimon.index.IndexFileMeta; +import org.apache.paimon.io.DataInputDeserializer; import org.apache.paimon.io.DataOutputSerializer; import org.junit.jupiter.api.Test; @@ -32,10 +33,42 @@ /** Tests for {@link PrimaryKeyIndexSourceMeta}. */ class PrimaryKeyIndexSourceMetaTest { + @Test + void testSerializedVersionRemainsOne() throws Exception { + PrimaryKeyIndexSourceMeta metadata = + new PrimaryKeyIndexSourceMeta(1, new PrimaryKeyIndexSourceFile("data-1", 10)); + + DataInputDeserializer input = new DataInputDeserializer(metadata.serialize()); + + assertThat(input.readInt()).isEqualTo(1); + } + + @Test + void testDataLevelRoundTrip() { + PrimaryKeyIndexSourceMeta metadata = + new PrimaryKeyIndexSourceMeta(3, new PrimaryKeyIndexSourceFile("data-1", 10)); + + PrimaryKeyIndexSourceMeta restored = + PrimaryKeyIndexSourceMeta.deserialize(metadata.serialize()); + + assertThat(restored.dataLevel()).isEqualTo(3); + assertThat(restored.sourceFiles()).isEqualTo(metadata.sourceFiles()); + } + + @Test + void testRejectsNonPositiveDataLevel() { + assertThatThrownBy( + () -> + new PrimaryKeyIndexSourceMeta( + 0, new PrimaryKeyIndexSourceFile("data-1", 10))) + .hasMessageContaining("data level must be positive"); + } + @Test void testMultipleSourceRoundTrip() { PrimaryKeyIndexSourceMeta metadata = new PrimaryKeyIndexSourceMeta( + 1, Arrays.asList( new PrimaryKeyIndexSourceFile("data-1", 10), new PrimaryKeyIndexSourceFile("data-2", 20))); @@ -51,7 +84,7 @@ void testMultipleSourceRoundTrip() { @Test void testSingleSourceRoundTrip() { PrimaryKeyIndexSourceMeta metadata = - new PrimaryKeyIndexSourceMeta(new PrimaryKeyIndexSourceFile("data-1", 0)); + new PrimaryKeyIndexSourceMeta(1, new PrimaryKeyIndexSourceFile("data-1", 0)); PrimaryKeyIndexSourceMeta restored = PrimaryKeyIndexSourceMeta.deserialize(metadata.serialize()); @@ -81,6 +114,7 @@ void testRejectsUnsupportedVersion() throws Exception { void testRejectsSourceCountBeforeAllocation() throws Exception { DataOutputSerializer output = new DataOutputSerializer(8); output.writeInt(1); + output.writeInt(1); output.writeInt(Integer.MAX_VALUE); assertThatThrownBy(() -> PrimaryKeyIndexSourceMeta.deserialize(output.getCopyOfBuffer())) @@ -94,6 +128,7 @@ void testRejectsTruncatedAndTrailingMetadata() throws Exception { DataOutputSerializer truncated = new DataOutputSerializer(64); truncated.writeInt(1); truncated.writeInt(1); + truncated.writeInt(1); truncated.writeUTF("data-1"); assertThatThrownBy(() -> PrimaryKeyIndexSourceMeta.deserialize(truncated.getCopyOfBuffer())) .hasMessageContaining("Failed to deserialize index source metadata"); @@ -101,6 +136,7 @@ void testRejectsTruncatedAndTrailingMetadata() throws Exception { DataOutputSerializer trailing = new DataOutputSerializer(64); trailing.writeInt(1); trailing.writeInt(1); + trailing.writeInt(1); trailing.writeUTF("data-1"); trailing.writeLong(1); trailing.writeByte(42); @@ -111,7 +147,7 @@ void testRejectsTruncatedAndTrailingMetadata() throws Exception { @Test void testReadsFromIndexFile() { PrimaryKeyIndexSourceMeta metadata = - new PrimaryKeyIndexSourceMeta(new PrimaryKeyIndexSourceFile("data-1", 5)); + new PrimaryKeyIndexSourceMeta(1, new PrimaryKeyIndexSourceFile("data-1", 5)); IndexFileMeta indexFile = new IndexFileMeta( "btree", diff --git a/paimon-core/src/test/java/org/apache/paimon/index/pkfulltext/BucketedFullTextIndexMaintainerTest.java b/paimon-core/src/test/java/org/apache/paimon/index/pkfulltext/BucketedFullTextIndexMaintainerTest.java index 0036da2d87fc..6a3630fb78b8 100644 --- a/paimon-core/src/test/java/org/apache/paimon/index/pkfulltext/BucketedFullTextIndexMaintainerTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/index/pkfulltext/BucketedFullTextIndexMaintainerTest.java @@ -92,7 +92,7 @@ void testReplacesArchiveWithCompactSourceWithoutMergingArchives() throws Excepti } @Test - void testMergesArchivesAtConfiguredFanout() throws Exception { + void testRebuildsDuplicateLevelArchivesAsCompleteLevel() throws Exception { DataFileMeta first = dataFile("data-1"); DataFileMeta second = dataFile("data-2"); IndexFileMeta firstPayload = payload("payload-1", first); @@ -105,8 +105,6 @@ void testMergesArchivesAtConfiguredFanout() throws Exception { 7, mock(PkFullTextIndexFile.class), builder, - 2, - 0.5, Arrays.asList(first, second), Arrays.asList(firstPayload, secondPayload), executor); @@ -137,8 +135,6 @@ void testRebuildsArchiveAtConfiguredStaleRatio() throws Exception { 7, mock(PkFullTextIndexFile.class), builder, - 5, - 0.5, Collections.singletonList(active), Collections.singletonList(oldPayload), executor); @@ -331,7 +327,7 @@ private static IndexFileMeta payload(String payloadName, List sour sourceFiles.add(new PrimaryKeyIndexSourceFile(source.fileName(), source.rowCount())); rowCount += source.rowCount(); } - PrimaryKeyIndexSourceMeta sourceMeta = new PrimaryKeyIndexSourceMeta(sourceFiles); + PrimaryKeyIndexSourceMeta sourceMeta = new PrimaryKeyIndexSourceMeta(1, sourceFiles); return new IndexFileMeta( "full-text", payloadName, diff --git a/paimon-core/src/test/java/org/apache/paimon/index/pkfulltext/PkFullTextBucketIndexStateTest.java b/paimon-core/src/test/java/org/apache/paimon/index/pkfulltext/PkFullTextBucketIndexStateTest.java index a2a988c2a98f..a72fe471f1c8 100644 --- a/paimon-core/src/test/java/org/apache/paimon/index/pkfulltext/PkFullTextBucketIndexStateTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/index/pkfulltext/PkFullTextBucketIndexStateTest.java @@ -41,8 +41,7 @@ void testClassifiesMatchingFieldAndRetiresOtherFields() { IndexFileMeta otherField = payload("other", "third-data", 8); PkFullTextBucketIndexState state = - PkFullTextBucketIndexState.fromActivePayloads( - 7, Arrays.asList(current, alsoCurrent, otherField)); + new PkFullTextBucketIndexState(7, Arrays.asList(current, alsoCurrent, otherField)); assertThat(state.currentPayloads()).containsExactly(current, alsoCurrent); assertThat(state.stalePayloads()).containsExactly(otherField); @@ -53,7 +52,7 @@ void testClassifiesMatchingFieldAndRetiresOtherFields() { void testRejectsDuplicateCurrentCoverage() { assertThatThrownBy( () -> - PkFullTextBucketIndexState.fromActivePayloads( + new PkFullTextBucketIndexState( 7, Arrays.asList( payload("first", "data", 7), @@ -66,14 +65,14 @@ void testRejectsDuplicateCurrentCoverage() { void testMapsEverySourceOfMultiSourceArchive() { PrimaryKeyIndexSourceMeta sourceMeta = new PrimaryKeyIndexSourceMeta( + 1, Arrays.asList( new PrimaryKeyIndexSourceFile("data-1", 1), new PrimaryKeyIndexSourceFile("data-2", 1))); IndexFileMeta payload = payload("multi", 7, 2, sourceMeta); PkFullTextBucketIndexState state = - PkFullTextBucketIndexState.fromActivePayloads( - 7, Collections.singletonList(payload)); + new PkFullTextBucketIndexState(7, Collections.singletonList(payload)); assertThat(state.currentPayloads()).containsExactly(payload); assertThat(state.payloadBySourceFile()) @@ -86,7 +85,7 @@ private static IndexFileMeta payload(String payloadName, String sourceName, int payloadName, fieldId, 1, - new PrimaryKeyIndexSourceMeta(new PrimaryKeyIndexSourceFile(sourceName, 1))); + new PrimaryKeyIndexSourceMeta(1, new PrimaryKeyIndexSourceFile(sourceName, 1))); } private static IndexFileMeta payload( diff --git a/paimon-core/src/test/java/org/apache/paimon/index/pkfulltext/PkFullTextIndexFileTest.java b/paimon-core/src/test/java/org/apache/paimon/index/pkfulltext/PkFullTextIndexFileTest.java index f7b3d82690d3..ffbfab5ff621 100644 --- a/paimon-core/src/test/java/org/apache/paimon/index/pkfulltext/PkFullTextIndexFileTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/index/pkfulltext/PkFullTextIndexFileTest.java @@ -66,7 +66,7 @@ class PkFullTextIndexFileTest { @TempDir java.nio.file.Path tempPath; @Test - void testWritesEveryPhysicalOrdinalAndV1SourceMetadata() throws Exception { + void testWritesEveryPhysicalOrdinalAndDataLevelMetadata() throws Exception { LocalFileIO fileIO = LocalFileIO.create(); PkFullTextDataFileReader reader = mock(PkFullTextDataFileReader.class); when(reader.rowCount()).thenReturn(3L); @@ -96,6 +96,7 @@ void testWritesEveryPhysicalOrdinalAndV1SourceMetadata() throws Exception { new DataInputDeserializer(archive.globalIndexMeta().sourceMeta()); assertThat(sourceInput.readInt()).isEqualTo(1); PrimaryKeyIndexSourceMeta sourceMeta = PrimaryKeyIndexSourceMeta.fromIndexFile(archive); + assertThat(sourceMeta.dataLevel()).isEqualTo(3); assertThat(sourceMeta.sourceFile().fileName()).isEqualTo("data-1"); assertThat(sourceMeta.sourceFile().rowCount()).isEqualTo(3); } @@ -201,20 +202,21 @@ void testClosesReaderWhenIndexerCreationFails() throws Exception { private static DataFileMeta dataFile(String fileName, long rowCount) { return DataFileMeta.forAppend( - fileName, - 100, - rowCount, - SimpleStats.EMPTY_STATS, - 0, - 1, - 1, - Collections.emptyList(), - null, - FileSource.COMPACT, - null, - null, - null, - null); + fileName, + 100, + rowCount, + SimpleStats.EMPTY_STATS, + 0, + 1, + 1, + Collections.emptyList(), + null, + FileSource.COMPACT, + null, + null, + null, + null) + .upgrade(3); } private IndexPathFactory pathFactory() { diff --git a/paimon-core/src/test/java/org/apache/paimon/index/pkfulltext/PrimaryKeyFullTextBucketSearchTest.java b/paimon-core/src/test/java/org/apache/paimon/index/pkfulltext/PrimaryKeyFullTextBucketSearchTest.java index 09d0df4a1513..d5e148341e11 100644 --- a/paimon-core/src/test/java/org/apache/paimon/index/pkfulltext/PrimaryKeyFullTextBucketSearchTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/index/pkfulltext/PrimaryKeyFullTextBucketSearchTest.java @@ -387,7 +387,7 @@ private static IndexFileMeta payload(String source, String name) { private static IndexFileMeta payload(String source, String name, long rowCount) { byte[] sourceMeta = - new PrimaryKeyIndexSourceMeta(new PrimaryKeyIndexSourceFile(source, rowCount)) + new PrimaryKeyIndexSourceMeta(1, new PrimaryKeyIndexSourceFile(source, rowCount)) .serialize(); return new IndexFileMeta( "full-text", @@ -403,7 +403,7 @@ private static IndexFileMeta payload(List sources, String name) { for (String source : sources) { sourceFiles.add(new PrimaryKeyIndexSourceFile(source, 3)); } - byte[] sourceMeta = new PrimaryKeyIndexSourceMeta(sourceFiles).serialize(); + byte[] sourceMeta = new PrimaryKeyIndexSourceMeta(1, sourceFiles).serialize(); long rowCount = 3L * sourceFiles.size(); return new IndexFileMeta( "full-text", diff --git a/paimon-core/src/test/java/org/apache/paimon/index/pksorted/BucketedSortedIndexMaintainerTest.java b/paimon-core/src/test/java/org/apache/paimon/index/pksorted/BucketedSortedIndexMaintainerTest.java index 3dcbb1bc1993..2b1fcffffe9d 100644 --- a/paimon-core/src/test/java/org/apache/paimon/index/pksorted/BucketedSortedIndexMaintainerTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/index/pksorted/BucketedSortedIndexMaintainerTest.java @@ -70,8 +70,7 @@ void shutdownExecutor() { void testRestoreBuildAndSourceRemoval() throws Exception { DataFileMeta oldSource = dataFile("data-1", 3); DataFileMeta newSource = dataFile("data-2", 3); - List oldPayloads = - Arrays.asList(payload("old-1", oldSource, 2), payload("old-2", oldSource, 1)); + List oldPayloads = Collections.singletonList(payload("old", oldSource, 3)); IndexFileMeta newPayload = payload("new", newSource, 3); PkSortedIndexFile indexFile = new PkSortedIndexFile(LocalFileIO.create(), pathFactory()); BucketedSortedIndexMaintainer maintainer = @@ -83,8 +82,6 @@ void testRestoreBuildAndSourceRemoval() throws Exception { assertThat(sourceFiles).containsExactly(newSource); return newPayload; }, - 5, - 0.2, Collections.singletonList(oldSource), oldPayloads, executor); @@ -108,7 +105,7 @@ void testRestoreBuildAndSourceRemoval() throws Exception { } @Test - void testFanoutCompactionReplacesCompleteGroups() throws Exception { + void testRebuildsDuplicateLevelPayloadsAsCompleteLevel() throws Exception { DataFileMeta sourceA = dataFile("data-a", 3); DataFileMeta sourceB = dataFile("data-b", 3); IndexFileMeta payloadA = payload("index-a", sourceA, 3); @@ -123,8 +120,6 @@ void testFanoutCompactionReplacesCompleteGroups() throws Exception { assertThat(sources).containsExactly(sourceA, sourceB); return merged; }, - 2, - 1.0, Arrays.asList(sourceA, sourceB), Arrays.asList(payloadA, payloadB), executor); @@ -156,8 +151,6 @@ void testCoveredFanoutIsReportedAsPendingMaintenance() { sources -> { throw new AssertionError("Pending build must not start."); }, - 2, - 1.0, Arrays.asList(sourceA, sourceB), Arrays.asList( payload("index-a", sourceA, 3), payload("index-b", sourceB, 3)), @@ -182,8 +175,6 @@ void testPartiallyStaleGroupRebuildsOnlyActiveSources() throws Exception { assertThat(sourceFiles).containsExactly(activeSource); return rebuiltPayload; }, - 5, - 0.3, Arrays.asList(staleSource, activeSource), Collections.singletonList(oldPayload), executor); @@ -219,8 +210,6 @@ void testAllStaleGroupIsDeletedWithoutBuild() throws Exception { builds.incrementAndGet(); return oldPayload; }, - 5, - 0.2, Collections.singletonList(source), Collections.singletonList(oldPayload), executor); @@ -242,7 +231,7 @@ void testAllStaleGroupIsDeletedWithoutBuild() throws Exception { } @Test - void testBlockingFanoutDeletesIntermediateGeneratedGroups() throws Exception { + void testBlockingBuildsCompleteLevelWithoutIntermediates() throws Exception { List sources = Arrays.asList( dataFile("data-a", 3), @@ -267,8 +256,6 @@ void testBlockingFanoutDeletesIntermediateGeneratedGroups() throws Exception { generated.add(payload); return payload; }, - 2, - 1.0, Collections.emptyList(), Collections.emptyList(), executor); @@ -280,13 +267,12 @@ void testBlockingFanoutDeletesIntermediateGeneratedGroups() throws Exception { Collections.emptyList(), sources, Collections.emptyList()), true); - assertThat(generated).hasSize(7); + assertThat(generated).hasSize(1); assertThat(commit.compactIncrement()).isPresent(); assertThat(commit.compactIncrement().get().newIndexFiles()) - .containsExactly(generated.get(6)); + .containsExactly(generated.get(0)); assertThat(commit.compactIncrement().get().deletedIndexFiles()).isEmpty(); - assertThat(indexFile.deleted()) - .containsExactlyInAnyOrderElementsOf(generated.subList(0, 6)); + assertThat(indexFile.deleted()).isEmpty(); } @Test @@ -302,8 +288,6 @@ void testRestoreDeletesInvalidPayloadsBeforePublishingReplacement() throws Excep "btree", indexFile, sourceFiles -> replacementPayload, - 5, - 0.2, Collections.singletonList(source), invalidPayloads, executor); @@ -326,8 +310,6 @@ void testRestoreDeletesInvalidPayloadsBeforePublishingReplacement() throws Excep sourceFiles -> { throw new AssertionError("Covered source must not be rebuilt."); }, - 5, - 0.2, Collections.singletonList(source), Collections.singletonList(replacementPayload), executor); @@ -348,8 +330,7 @@ void testRestoreDeletesInvalidPayloadsBeforePublishingReplacement() throws Excep void testFinalBuildFailureThrowsAndRollsBackSourceTransition() { DataFileMeta oldSource = dataFile("data-1", 3); DataFileMeta newSource = dataFile("data-2", 3); - List oldPayloads = - Arrays.asList(payload("old-1", oldSource, 2), payload("old-2", oldSource, 1)); + List oldPayloads = Collections.singletonList(payload("old", oldSource, 3)); AtomicInteger attempts = new AtomicInteger(); BucketedSortedIndexMaintainer maintainer = new BucketedSortedIndexMaintainer( @@ -360,8 +341,6 @@ void testFinalBuildFailureThrowsAndRollsBackSourceTransition() { attempts.incrementAndGet(); throw new IllegalStateException("expected build failure"); }, - 5, - 0.2, Collections.singletonList(oldSource), oldPayloads, executor); @@ -400,8 +379,6 @@ void testTransientFailureRetriesAndPublishesWholeGroup() throws Exception { } return payload; }, - 5, - 0.2, Collections.emptyList(), Collections.emptyList(), executor); @@ -433,8 +410,6 @@ void testNonBlockingBuildPublishesOnLaterCommit() throws Exception { release.await(); return payload; }, - 5, - 0.2, Collections.emptyList(), Collections.emptyList(), executor); @@ -457,6 +432,36 @@ void testNonBlockingBuildPublishesOnLaterCommit() throws Exception { assertThat(maintainer.buildNotCompleted()).isFalse(); } + @Test + void testDoesNotPublishPayloadForDifferentDataLevel() throws Exception { + DataFileMeta source = dataFile("data-1", 3); + IndexFileMeta wrongLevelPayload = + payload("wrong-level", Collections.singletonList(source), 3, 2); + TrackingPkSortedIndexFile indexFile = + new TrackingPkSortedIndexFile(LocalFileIO.create(), pathFactory()); + BucketedSortedIndexMaintainer maintainer = + new BucketedSortedIndexMaintainer( + 7, + "btree", + indexFile, + sourceFiles -> wrongLevelPayload, + Collections.emptyList(), + Collections.emptyList(), + executor); + + maintainer.prepareCommit(DataIncrement.emptyIncrement(), compactAfter(source), false, true); + executor.submit(() -> {}).get(5, TimeUnit.SECONDS); + BucketedSortedIndexMaintainer.SortedIndexCommit commit = + maintainer.prepareCommit( + DataIncrement.emptyIncrement(), + CompactIncrement.emptyIncrement(), + false, + false); + + assertThat(commit.appendIncrement()).isEmpty(); + assertThat(indexFile.deleted()).containsExactly(wrongLevelPayload); + } + @Test void testStaleCompletionIsDeletedBeforeReplacementPublishes() throws Exception { DataFileMeta staleSource = dataFile("data-1", 3); @@ -484,8 +489,6 @@ void testStaleCompletionIsDeletedBeforeReplacementPublishes() throws Exception { assertThat(sourceFile).isEqualTo(activeSource); return activePayload; }, - 5, - 0.2, Collections.emptyList(), Collections.emptyList(), executor); @@ -514,11 +517,10 @@ void testStaleCompletionIsDeletedBeforeReplacementPublishes() throws Exception { } @Test - void testFanoutCompletionIsDiscardedWhenPlannedSourceRetires() throws Exception { + void testLevelCompletionIsDiscardedWhenPlannedSourceRetires() throws Exception { DataFileMeta sourceA = dataFile("data-a", 3); DataFileMeta sourceB = dataFile("data-b", 3); IndexFileMeta payloadA = payload("index-a", sourceA, 3); - IndexFileMeta payloadB = payload("index-b", sourceB, 3); IndexFileMeta staleMerged = payload("index-ab", Arrays.asList(sourceA, sourceB), 6); CountDownLatch started = new CountDownLatch(1); CountDownLatch release = new CountDownLatch(1); @@ -534,14 +536,11 @@ void testFanoutCompletionIsDiscardedWhenPlannedSourceRetires() throws Exception release.await(); return staleMerged; }, - 2, - 1.0, - Arrays.asList(sourceA, sourceB), - Arrays.asList(payloadA, payloadB), + Collections.singletonList(sourceA), + Collections.singletonList(payloadA), executor); - maintainer.prepareCommit( - DataIncrement.emptyIncrement(), CompactIncrement.emptyIncrement(), false); + maintainer.prepareCommit(DataIncrement.emptyIncrement(), compactAfter(sourceB), false); assertThat(started.await(5, TimeUnit.SECONDS)).isTrue(); maintainer.prepareCommit( DataIncrement.emptyIncrement(), @@ -558,9 +557,7 @@ void testFanoutCompletionIsDiscardedWhenPlannedSourceRetires() throws Exception DataIncrement.emptyIncrement(), CompactIncrement.emptyIncrement(), true); assertThat(indexFile.deleted()).contains(staleMerged); - assertThat(cleanup.appendIncrement()).isPresent(); - assertThat(cleanup.appendIncrement().get().newIndexFiles()).isEmpty(); - assertThat(cleanup.appendIncrement().get().deletedIndexFiles()).containsExactly(payloadB); + assertThat(cleanup.appendIncrement()).isEmpty(); assertThat(maintainer.state().groups()).hasSize(1); assertThat(maintainer.state().groups().get(0).payloads()).containsExactly(payloadA); } @@ -576,8 +573,6 @@ void testRejectedSubmissionThrowsAndRollsBackSourceTransition() { "btree", new PkSortedIndexFile(LocalFileIO.create(), pathFactory()), sourceFiles -> payload("new", source, 3), - 5, - 0.2, Collections.emptyList(), Collections.emptyList(), rejectedExecutor); @@ -606,8 +601,6 @@ void testMalformedOutputThrowsAndRollsBackSourceTransition() { "btree", indexFile, sourceFiles -> malformed, - 5, - 0.2, Collections.emptyList(), Collections.emptyList(), executor); @@ -650,8 +643,6 @@ void testCloseDeletesResultCompletedAfterCancellation() throws Exception { } return generated; }, - 5, - 0.2, Collections.emptyList(), Collections.emptyList(), executor); @@ -670,8 +661,7 @@ void testCloseDeletesResultCompletedAfterCancellation() throws Exception { void testInterruptedCommitRollsBackStateAndCancelsBuild() throws Exception { DataFileMeta oldSource = dataFile("data-1", 3); DataFileMeta newSource = dataFile("data-2", 3); - List oldPayloads = - Arrays.asList(payload("old-1", oldSource, 2), payload("old-2", oldSource, 1)); + List oldPayloads = Collections.singletonList(payload("old", oldSource, 3)); CountDownLatch started = new CountDownLatch(1); CountDownLatch cancelled = new CountDownLatch(1); BucketedSortedIndexMaintainer maintainer = @@ -689,8 +679,6 @@ void testInterruptedCommitRollsBackStateAndCancelsBuild() throws Exception { throw e; } }, - 5, - 0.2, Collections.singletonList(oldSource), oldPayloads, executor); @@ -739,6 +727,11 @@ private static IndexFileMeta payload( private static IndexFileMeta payload( String fileName, List sourceFiles, long payloadRowCount) { + return payload(fileName, sourceFiles, payloadRowCount, sourceFiles.get(0).level()); + } + + private static IndexFileMeta payload( + String fileName, List sourceFiles, long payloadRowCount, int dataLevel) { List sources = new java.util.ArrayList<>(); long rowCount = 0; for (DataFileMeta sourceFile : sourceFiles) { @@ -757,7 +750,7 @@ private static IndexFileMeta payload( 7, null, new byte[] {1}, - new PrimaryKeyIndexSourceMeta(sources).serialize()), + new PrimaryKeyIndexSourceMeta(dataLevel, sources).serialize()), null); } diff --git a/paimon-core/src/test/java/org/apache/paimon/index/pksorted/PkSortedBucketIndexStateTest.java b/paimon-core/src/test/java/org/apache/paimon/index/pksorted/PkSortedBucketIndexStateTest.java index 758937b40e6c..f7a9a60d1dc3 100644 --- a/paimon-core/src/test/java/org/apache/paimon/index/pksorted/PkSortedBucketIndexStateTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/index/pksorted/PkSortedBucketIndexStateTest.java @@ -22,6 +22,9 @@ import org.apache.paimon.index.IndexFileMeta; import org.apache.paimon.index.pk.PrimaryKeyIndexSourceFile; import org.apache.paimon.index.pk.PrimaryKeyIndexSourceMeta; +import org.apache.paimon.io.DataFileMeta; +import org.apache.paimon.manifest.FileSource; +import org.apache.paimon.stats.SimpleStats; import org.junit.jupiter.api.Test; @@ -35,299 +38,143 @@ class PkSortedBucketIndexStateTest { @Test - void testRotatedPayloadsFormOneCoveredGroup() { - PrimaryKeyIndexSourceFile source = new PrimaryKeyIndexSourceFile("data-1", 10); - PkSortedBucketIndexState state = - PkSortedBucketIndexState.fromActivePayloads( - 7, - "btree", - Collections.singletonList(source), - Arrays.asList( - payload("index-1", source, "btree", 7, 0, 9, 4), - payload("index-2", source, "btree", 7, 0, 9, 6))); - - assertThat(state.groups()).hasSize(1); - assertThat(state.groups().get(0).payloads()) - .extracting(IndexFileMeta::fileName) - .containsExactly("index-1", "index-2"); - assertThat(state.coveredSourceFiles()).containsExactly(source); - assertThat(state.uncoveredSourceFiles()).isEmpty(); - assertThat(state.rejectedPayloads()).isEmpty(); - } - - @Test - void testMultiSourcePayloadsFormOneCoveredGroup() { - PrimaryKeyIndexSourceFile sourceA = new PrimaryKeyIndexSourceFile("data-a", 3); - PrimaryKeyIndexSourceFile sourceB = new PrimaryKeyIndexSourceFile("data-b", 7); - List sources = Arrays.asList(sourceA, sourceB); - PkSortedBucketIndexState state = - PkSortedBucketIndexState.fromActivePayloads( - 7, - "btree", - sources, - Arrays.asList( - payload("index-1", sources, "btree", 7, 0, 9, 4), - payload("index-2", sources, "btree", 7, 0, 9, 6))); - - assertThat(state.groups()).hasSize(1); - assertThat(state.groups().get(0).sourceFiles()).containsExactly(sourceA, sourceB); - assertThat(state.coveredSourceFiles()).containsExactly(sourceA, sourceB); - assertThat(state.uncoveredSourceFiles()).isEmpty(); - assertThat(state.rejectedPayloads()).isEmpty(); - } - - @Test - void testPartiallyStaleGroupRemainsAndCoversItsActiveSource() { - PrimaryKeyIndexSourceFile stale = new PrimaryKeyIndexSourceFile("data-a", 3); - PrimaryKeyIndexSourceFile active = new PrimaryKeyIndexSourceFile("data-b", 7); - List sources = Arrays.asList(stale, active); + void testAcceptsOnePayloadForCompleteLevel() { + DataFileMeta first = dataFile("data-a", 3, 2); + DataFileMeta second = dataFile("data-b", 7, 2); + IndexFileMeta payload = payload("index", 2, first, second); PkSortedBucketIndexState state = - PkSortedBucketIndexState.fromActivePayloads( + PkSortedBucketIndexState.fromActiveDataFiles( 7, "btree", - Collections.singletonList(active), - Collections.singletonList( - payload("index-1", sources, "btree", 7, 0, 9, 10))); + Arrays.asList(second, first), + Collections.singletonList(payload)); assertThat(state.groups()).hasSize(1); - assertThat(state.groups().get(0).sourceFiles()).containsExactly(stale, active); - assertThat(state.coveredSourceFiles()).containsExactly(active); + assertThat(state.groups().get(0).dataLevel()).isEqualTo(2); + assertThat(state.groups().get(0).sourceFiles()) + .extracting(PrimaryKeyIndexSourceFile::fileName) + .containsExactly("data-a", "data-b"); + assertThat(state.coveredSourceFiles()).hasSize(2); assertThat(state.uncoveredSourceFiles()).isEmpty(); assertThat(state.rejectedPayloads()).isEmpty(); } @Test - void testOverlappingActiveSourceRejectsLaterGroup() { - PrimaryKeyIndexSourceFile sourceA = new PrimaryKeyIndexSourceFile("data-a", 3); - PrimaryKeyIndexSourceFile sourceB = new PrimaryKeyIndexSourceFile("data-b", 7); - - IndexFileMeta first = - payload("index-ab", Arrays.asList(sourceA, sourceB), "btree", 7, 0, 9, 10); - IndexFileMeta overlapping = payload("index-b", sourceB, "btree", 7, 0, 6, 7); - PkSortedBucketIndexState state = - PkSortedBucketIndexState.fromActivePayloads( - 7, - "btree", - Arrays.asList(sourceA, sourceB), - Arrays.asList(first, overlapping)); - - assertThat(state.groups()).hasSize(1); - assertThat(state.groups().get(0).payloads()).containsExactly(first); - assertThat(state.coveredSourceFiles()).containsExactly(sourceA, sourceB); - assertThat(state.uncoveredSourceFiles()).isEmpty(); - assertThat(state.rejectedPayloads()).containsExactly(overlapping); - } - - @Test - void testDuplicateSourcesRejectWholeGroup() { - PrimaryKeyIndexSourceFile source = new PrimaryKeyIndexSourceFile("data-a", 3); - List duplicateSources = Arrays.asList(source, source); - IndexFileMeta duplicated = payload("index-a", duplicateSources, "btree", 7, 0, 5, 6); - - PkSortedBucketIndexState state = - PkSortedBucketIndexState.fromActivePayloads( - 7, - "btree", - Collections.singletonList(source), - Collections.singletonList(duplicated)); - - assertThat(state.groups()).isEmpty(); - assertThat(state.uncoveredSourceFiles()).containsExactly(source); - assertThat(state.rejectedPayloads()).containsExactly(duplicated); - } - - @Test - void testSourceRowCountOverflowRejectsWholeGroup() { - PrimaryKeyIndexSourceFile huge = new PrimaryKeyIndexSourceFile("data-a", Long.MAX_VALUE); - PrimaryKeyIndexSourceFile extra = new PrimaryKeyIndexSourceFile("data-b", 1); - IndexFileMeta overflowing = - payload( - "index-overflow", - Arrays.asList(huge, extra), - "btree", - 7, - 0, - Long.MAX_VALUE, - 1); + void testRejectsPartialLevelPayload() { + DataFileMeta first = dataFile("data-a", 3, 2); + DataFileMeta second = dataFile("data-b", 7, 2); + IndexFileMeta partial = payload("partial", 2, first); PkSortedBucketIndexState state = - PkSortedBucketIndexState.fromActivePayloads( + PkSortedBucketIndexState.fromActiveDataFiles( 7, "btree", - Arrays.asList(huge, extra), - Collections.singletonList(overflowing)); + Arrays.asList(first, second), + Collections.singletonList(partial)); assertThat(state.groups()).isEmpty(); - assertThat(state.uncoveredSourceFiles()).containsExactly(huge, extra); - assertThat(state.rejectedPayloads()).containsExactly(overflowing); + assertThat(state.uncoveredSourceFiles()).hasSize(2); + assertThat(state.rejectedPayloads()).containsExactly(partial); } @Test - void testIncompletePayloadRowCountLeavesSourceUncovered() { - PrimaryKeyIndexSourceFile source = new PrimaryKeyIndexSourceFile("data-1", 10); - PkSortedBucketIndexState state = - PkSortedBucketIndexState.fromActivePayloads( - 7, - "btree", - Collections.singletonList(source), - Arrays.asList( - payload("index-1", source, "btree", 7, 0, 9, 4), - payload("index-2", source, "btree", 7, 0, 9, 5))); + void testRejectsDuplicatePayloadsForLevel() { + DataFileMeta data = dataFile("data", 3, 2); + IndexFileMeta first = payload("first", 2, data); + IndexFileMeta second = payload("second", 2, data); - assertThat(state.groups()).isEmpty(); - assertThat(state.coveredSourceFiles()).isEmpty(); - assertThat(state.uncoveredSourceFiles()).containsExactly(source); - assertThat(state.rejectedPayloads()) - .extracting(IndexFileMeta::fileName) - .containsExactly("index-1", "index-2"); - } - - @Test - void testWrongPayloadRangeLeavesSourceUncovered() { - PrimaryKeyIndexSourceFile source = new PrimaryKeyIndexSourceFile("data-1", 10); PkSortedBucketIndexState state = - PkSortedBucketIndexState.fromActivePayloads( - 7, - "bitmap", - Collections.singletonList(source), - Arrays.asList( - payload("index-1", source, "bitmap", 7, 0, 9, 4), - payload("index-2", source, "bitmap", 7, 0, 8, 6))); + PkSortedBucketIndexState.fromActiveDataFiles( + 7, "btree", Collections.singletonList(data), Arrays.asList(first, second)); assertThat(state.groups()).isEmpty(); - assertThat(state.coveredSourceFiles()).isEmpty(); - assertThat(state.uncoveredSourceFiles()).containsExactly(source); + assertThat(state.rejectedPayloads()).containsExactly(first, second); } @Test - void testMixedIndexTypeLeavesSourceUncovered() { - PrimaryKeyIndexSourceFile source = new PrimaryKeyIndexSourceFile("data-1", 10); - PkSortedBucketIndexState state = - PkSortedBucketIndexState.fromActivePayloads( - 7, - "btree", - Collections.singletonList(source), - Arrays.asList( - payload("index-1", source, "btree", 7, 0, 9, 4), - payload("index-2", source, "bitmap", 7, 0, 9, 6))); - - assertThat(state.groups()).isEmpty(); - assertThat(state.coveredSourceFiles()).isEmpty(); - assertThat(state.uncoveredSourceFiles()).containsExactly(source); - } + void testRejectsPayloadForDifferentLevel() { + DataFileMeta data = dataFile("data", 3, 2); + IndexFileMeta wrongLevel = payload("wrong-level", 3, data); - @Test - void testMixedFieldLeavesSourceUncovered() { - PrimaryKeyIndexSourceFile source = new PrimaryKeyIndexSourceFile("data-1", 10); PkSortedBucketIndexState state = - PkSortedBucketIndexState.fromActivePayloads( + PkSortedBucketIndexState.fromActiveDataFiles( 7, "btree", - Collections.singletonList(source), - Arrays.asList( - payload("index-1", source, "btree", 7, 0, 9, 4), - payload("index-2", source, "btree", 8, 0, 9, 6))); + Collections.singletonList(data), + Collections.singletonList(wrongLevel)); assertThat(state.groups()).isEmpty(); - assertThat(state.coveredSourceFiles()).isEmpty(); - assertThat(state.uncoveredSourceFiles()).containsExactly(source); + assertThat(state.rejectedPayloads()).containsExactly(wrongLevel); } @Test - void testMismatchedSourceMetadataLeavesSourceUncovered() { - PrimaryKeyIndexSourceFile source = new PrimaryKeyIndexSourceFile("data-1", 10); - PrimaryKeyIndexSourceFile mismatchedSource = new PrimaryKeyIndexSourceFile("data-1", 11); - PkSortedBucketIndexState state = - PkSortedBucketIndexState.fromActivePayloads( - 7, - "btree", - Collections.singletonList(source), - Arrays.asList( - payload("index-1", source, "btree", 7, 0, 9, 4), - payload("index-2", mismatchedSource, "btree", 7, 0, 9, 6))); - - assertThat(state.groups()).isEmpty(); - assertThat(state.coveredSourceFiles()).isEmpty(); - assertThat(state.uncoveredSourceFiles()).containsExactly(source); - } - - @Test - void testDuplicatePayloadNameLeavesSourceUncovered() { - PrimaryKeyIndexSourceFile source = new PrimaryKeyIndexSourceFile("data-1", 10); - PkSortedBucketIndexState state = - PkSortedBucketIndexState.fromActivePayloads( - 7, - "btree", - Collections.singletonList(source), - Arrays.asList( - payload("index-1", source, "btree", 7, 0, 9, 5), - payload("index-1", source, "btree", 7, 0, 9, 5))); - - assertThat(state.groups()).isEmpty(); - assertThat(state.coveredSourceFiles()).isEmpty(); - assertThat(state.uncoveredSourceFiles()).containsExactly(source); - } - - @Test - void testMalformedSourceMetadataLeavesSourceUncovered() { - PrimaryKeyIndexSourceFile source = new PrimaryKeyIndexSourceFile("data-1", 10); + void testRejectsMalformedSourceMetadata() { + DataFileMeta data = dataFile("data", 3, 2); IndexFileMeta malformed = new IndexFileMeta( "btree", - "index-1", + "malformed", 100, - 10, - new GlobalIndexMeta(0, 9, 7, null, new byte[] {1}, new byte[] {1}), + 3, + new GlobalIndexMeta(0, 2, 7, null, new byte[] {1}, new byte[] {1}), null); PkSortedBucketIndexState state = - PkSortedBucketIndexState.fromActivePayloads( + PkSortedBucketIndexState.fromActiveDataFiles( 7, "btree", - Collections.singletonList(source), + Collections.singletonList(data), Collections.singletonList(malformed)); assertThat(state.groups()).isEmpty(); - assertThat(state.coveredSourceFiles()).isEmpty(); - assertThat(state.uncoveredSourceFiles()).containsExactly(source); assertThat(state.rejectedPayloads()).containsExactly(malformed); } - private static IndexFileMeta payload( - String fileName, - PrimaryKeyIndexSourceFile source, - String indexType, - int fieldId, - long rangeStart, - long rangeEnd, - long rowCount) { - return payload( - fileName, - Collections.singletonList(source), - indexType, - fieldId, - rangeStart, - rangeEnd, - rowCount); + private static DataFileMeta dataFile(String name, long rowCount, int level) { + return DataFileMeta.forAppend( + name, + 100, + rowCount, + SimpleStats.EMPTY_STATS, + 0, + 0, + 1, + Collections.emptyList(), + null, + FileSource.COMPACT, + null, + null, + null, + null) + .upgrade(level); } - private static IndexFileMeta payload( - String fileName, - java.util.List sources, - String indexType, - int fieldId, - long rangeStart, - long rangeEnd, - long rowCount) { - byte[] sourceMeta = new PrimaryKeyIndexSourceMeta(sources).serialize(); + private static IndexFileMeta payload(String name, int level, DataFileMeta... files) { + List sources = + Arrays.asList(files).stream() + .sorted(java.util.Comparator.comparing(DataFileMeta::fileName)) + .map( + file -> + new PrimaryKeyIndexSourceFile( + file.fileName(), file.rowCount())) + .collect(java.util.stream.Collectors.toList()); + long rowCount = 0; + for (PrimaryKeyIndexSourceFile source : sources) { + rowCount += source.rowCount(); + } return new IndexFileMeta( - indexType, - fileName, + "btree", + name, 100, rowCount, new GlobalIndexMeta( - rangeStart, rangeEnd, fieldId, null, new byte[] {1}, sourceMeta), + 0, + rowCount - 1, + 7, + null, + new byte[] {1}, + new PrimaryKeyIndexSourceMeta(level, sources).serialize()), null); } } diff --git a/paimon-core/src/test/java/org/apache/paimon/index/pksorted/PkSortedIndexBuilderTest.java b/paimon-core/src/test/java/org/apache/paimon/index/pksorted/PkSortedIndexBuilderTest.java index 917f013da72c..db2d6d0b949d 100644 --- a/paimon-core/src/test/java/org/apache/paimon/index/pksorted/PkSortedIndexBuilderTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/index/pksorted/PkSortedIndexBuilderTest.java @@ -34,6 +34,7 @@ import org.apache.paimon.index.IndexFileMeta; import org.apache.paimon.index.IndexPathFactory; import org.apache.paimon.index.pk.PrimaryKeyIndexSourceFile; +import org.apache.paimon.index.pk.PrimaryKeyIndexSourceMeta; import org.apache.paimon.io.DataFileMeta; import org.apache.paimon.manifest.FileSource; import org.apache.paimon.options.Options; @@ -104,6 +105,8 @@ void testBuildsQueryableBTreeAndBitmapFromUnsortedPhysicalRows() throws Exceptio assertQuery(fileIO, pathFactory, indexType, options, payload, false, 20, 0L, 3L); assertQuery(fileIO, pathFactory, indexType, options, payload, true, null, 1L); + assertThat(PrimaryKeyIndexSourceMeta.fromIndexFile(payload).dataLevel()) + .isEqualTo(source.level()); } } @@ -117,6 +120,7 @@ void testBuildsSeveralSourcesInDeterministicOrdinalOrder() throws Exception { new PkSortedIndexFile(LocalFileIO.create(), pathFactory(tempPath)) { @Override public IndexFileMeta build( + int dataLevel, List sourceFiles, DataField indexField, String indexType, @@ -172,6 +176,7 @@ void testForcedSpillSortsRowsAndClosesTaskOwnedIoManager() throws Exception { new PkSortedIndexFile(LocalFileIO.create(), pathFactory(tempPath)) { @Override public IndexFileMeta build( + int dataLevel, List sourceFiles, DataField indexField, String indexType, @@ -263,20 +268,21 @@ private static Options options() { private static DataFileMeta dataFile(String fileName, long rowCount) { return DataFileMeta.forAppend( - fileName, - 100, - rowCount, - SimpleStats.EMPTY_STATS, - 0, - 0, - 1, - Collections.emptyList(), - null, - FileSource.COMPACT, - null, - null, - null, - null); + fileName, + 100, + rowCount, + SimpleStats.EMPTY_STATS, + 0, + 0, + 1, + Collections.emptyList(), + null, + FileSource.COMPACT, + null, + null, + null, + null) + .upgrade(3); } private static IndexPathFactory pathFactory(java.nio.file.Path directory) { diff --git a/paimon-core/src/test/java/org/apache/paimon/index/pksorted/PkSortedIndexFileTest.java b/paimon-core/src/test/java/org/apache/paimon/index/pksorted/PkSortedIndexFileTest.java index 4720ade85b0d..e8e8775e1ee4 100644 --- a/paimon-core/src/test/java/org/apache/paimon/index/pksorted/PkSortedIndexFileTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/index/pksorted/PkSortedIndexFileTest.java @@ -101,6 +101,7 @@ void testBuildsSingleBTreeAndBitmapPayload() throws Exception { IndexFileMeta payload = indexFile.build( + 1, Collections.singletonList(source), field(), indexType, @@ -135,6 +136,7 @@ void testBuildsMultiSourcePayloadsInOneOrdinalDomain() throws Exception { IndexFileMeta payload = indexFile.build( + 1, sources, field(), "btree", @@ -196,6 +198,7 @@ public List finish() { assertThatThrownBy( () -> indexFile.build( + 1, Collections.singletonList( new PrimaryKeyIndexSourceFile("data-file", 2)), field(), @@ -246,6 +249,7 @@ protected GlobalIndexSingleColumnWriter createWriter( assertThatThrownBy( () -> indexFile.build( + 1, Collections.singletonList( new PrimaryKeyIndexSourceFile("data-file", 1)), field(), diff --git a/paimon-core/src/test/java/org/apache/paimon/index/pkvector/BucketedVectorIndexMaintainerTest.java b/paimon-core/src/test/java/org/apache/paimon/index/pkvector/BucketedVectorIndexMaintainerTest.java index 4d06fb5fefbb..3553f858ed50 100644 --- a/paimon-core/src/test/java/org/apache/paimon/index/pkvector/BucketedVectorIndexMaintainerTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/index/pkvector/BucketedVectorIndexMaintainerTest.java @@ -270,13 +270,12 @@ void testDiscardsStaleBuildAndIndexesLatestSources() throws Exception { } @Test - void testPartialCompactionKeepsOldAnnAndIndexesNewSource() throws Exception { + void testPartialCompactionRebuildsCompleteLevel() throws Exception { LocalFileIO fileIO = LocalFileIO.create(); PkVectorAnnSegmentFile annFile = new PkVectorAnnSegmentFile(fileIO, pathFactory()); DataField vectorField = new DataField(7, "embedding", DataTypes.VECTOR(2, DataTypes.FLOAT())); Options options = indexOptions(); - options.setString("fields.embedding.pk-index.compaction.stale-ratio-threshold", "1.0"); DataFileMeta data1 = dataFile("data-1"); DataFileMeta data2 = dataFile("data-2"); DataFileMeta data3 = dataFile("data-3"); @@ -320,14 +319,14 @@ data2, new ArrayReader(new float[][] {{2, 0}}))), BucketedVectorIndexMaintainer.VectorIndexIncrement increment = commit.compactIncrement().get(); - assertThat(increment.deletedIndexFiles()).isEmpty(); + assertThat(increment.deletedIndexFiles()).containsExactly(initialAnn); assertThat(increment.newIndexFiles()).hasSize(1); - IndexFileMeta delta = increment.newIndexFiles().get(0); - assertThat(delta.indexType()).isEqualTo("test-vector-ann"); - assertThat(PrimaryKeyIndexSourceMeta.fromIndexFile(delta).sourceFiles()) + IndexFileMeta replacement = increment.newIndexFiles().get(0); + assertThat(replacement.indexType()).isEqualTo("test-vector-ann"); + assertThat(PrimaryKeyIndexSourceMeta.fromIndexFile(replacement).sourceFiles()) .extracting(PrimaryKeyIndexSourceFile::fileName) - .containsExactly("data-3"); - assertThat(maintainer.segments()).containsExactly(initialAnn, delta); + .containsExactly("data-2", "data-3"); + assertThat(maintainer.segments()).containsExactly(replacement); } @Test @@ -467,13 +466,12 @@ void testPendingBuildUsesDeletionVectorSnapshot() throws Exception { } @Test - void testRebuildsDerivedLevelAndAtomicallyReplacesInputs() throws Exception { + void testRebuildsDuplicateLevelSegmentsAsCompleteLevel() throws Exception { LocalFileIO fileIO = LocalFileIO.create(); PkVectorAnnSegmentFile annFile = new PkVectorAnnSegmentFile(fileIO, pathFactory()); DataField vectorField = new DataField(7, "embedding", DataTypes.VECTOR(2, DataTypes.FLOAT())); Options options = indexOptions(); - options.setString("fields.embedding.pk-index.compaction.level-fanout", "3"); DataFileMeta data1 = dataFile("data-1"); DataFileMeta data2 = dataFile("data-2"); DataFileMeta data3 = dataFile("data-3"); @@ -546,7 +544,6 @@ void testPrepareCommitRollsBackMultipleRebuildsOnFailure() throws Exception { DataField vectorField = new DataField(7, "embedding", DataTypes.VECTOR(2, DataTypes.FLOAT())); Options options = indexOptions(); - options.setString("fields.embedding.pk-index.compaction.level-fanout", "3"); List dataFiles = new ArrayList<>(); List initialSegments = new ArrayList<>(); for (int i = 1; i <= 6; i++) { @@ -593,7 +590,7 @@ data, new ArrayReader(new float[][] {{(float) i, 0}}))), CompactIncrement.emptyIncrement(), true)) .isInstanceOf(UncheckedIOException.class); - assertThat(maintainer.segments()).containsExactlyInAnyOrderElementsOf(initialSegments); + assertThat(maintainer.segments()).isEmpty(); assertThat(fileCount()).isEqualTo(6); BucketedVectorIndexMaintainer.VectorIndexCommit retry = @@ -604,7 +601,7 @@ data, new ArrayReader(new float[][] {{(float) i, 0}}))), retry.appendIncrement().get(); assertThat(increment.deletedIndexFiles()) .containsExactlyInAnyOrderElementsOf(initialSegments); - assertThat(increment.newIndexFiles()).hasSize(2); + assertThat(increment.newIndexFiles()).hasSize(1); } @Test @@ -847,7 +844,7 @@ private static IndexFileMeta payload( fieldId, null, new byte[] {1}, - new PrimaryKeyIndexSourceMeta(sourceFile).serialize()), + new PrimaryKeyIndexSourceMeta(1, sourceFile).serialize()), null); } diff --git a/paimon-core/src/test/java/org/apache/paimon/index/pkvector/PkVectorAnnSegmentFileTest.java b/paimon-core/src/test/java/org/apache/paimon/index/pkvector/PkVectorAnnSegmentFileTest.java index 097ff80195c2..fd972d40c48e 100644 --- a/paimon-core/src/test/java/org/apache/paimon/index/pkvector/PkVectorAnnSegmentFileTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/index/pkvector/PkVectorAnnSegmentFileTest.java @@ -75,6 +75,7 @@ void testBuildSkipsNullAndExcludedPhysicalRows() throws Exception { assertThat(segment.indexType()).isEqualTo("test-vector-ann"); assertThat(segment.rowCount()).isEqualTo(1); PrimaryKeyIndexSourceMeta sourceMeta = PrimaryKeyIndexSourceMeta.fromIndexFile(segment); + assertThat(sourceMeta.dataLevel()).isEqualTo(3); assertThat(sourceMeta.sourceFiles()) .extracting(PrimaryKeyIndexSourceFile::fileName) .containsExactly("data-1"); @@ -308,20 +309,21 @@ private static Options indexOptions() { private static DataFileMeta dataFile(String fileName, long rowCount) { return DataFileMeta.forAppend( - fileName, - 100, - rowCount, - SimpleStats.EMPTY_STATS, - 0, - 0, - 1, - Collections.emptyList(), - null, - FileSource.COMPACT, - null, - null, - null, - null); + fileName, + 100, + rowCount, + SimpleStats.EMPTY_STATS, + 0, + 0, + 1, + Collections.emptyList(), + null, + FileSource.COMPACT, + null, + null, + null, + null) + .upgrade(3); } private IndexPathFactory pathFactory() { diff --git a/paimon-core/src/test/java/org/apache/paimon/index/pkvector/PkVectorBucketIndexStateTest.java b/paimon-core/src/test/java/org/apache/paimon/index/pkvector/PkVectorBucketIndexStateTest.java index be939cf8a000..04b88b7adb2d 100644 --- a/paimon-core/src/test/java/org/apache/paimon/index/pkvector/PkVectorBucketIndexStateTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/index/pkvector/PkVectorBucketIndexStateTest.java @@ -38,8 +38,7 @@ void testDerivesAnnCoverage() { IndexFileMeta ann = segment("ann", "data-1", "test-vector-ann"); PkVectorBucketIndexState state = - PkVectorBucketIndexState.fromActivePayloads( - 7, "test-vector-ann", Collections.singletonList(ann)); + new PkVectorBucketIndexState(7, "test-vector-ann", Collections.singletonList(ann)); assertThat(state.vectorFieldId()).isEqualTo(7); assertThat(state.annSegments()).extracting(IndexFileMeta::fileName).containsExactly("ann"); @@ -50,7 +49,7 @@ void testDerivesAnnCoverage() { void testRejectsDuplicateAnnSource() { assertThatThrownBy( () -> - PkVectorBucketIndexState.fromActivePayloads( + new PkVectorBucketIndexState( 7, "test-vector-ann", java.util.Arrays.asList( @@ -63,7 +62,7 @@ void testRejectsDuplicateAnnSource() { void testRejectsDifferentIndexType() { assertThatThrownBy( () -> - PkVectorBucketIndexState.fromActivePayloads( + new PkVectorBucketIndexState( 7, "test-vector-ann", Collections.singletonList( @@ -74,8 +73,7 @@ void testRejectsDifferentIndexType() { @Test void testEmptyPayloadsProduceEmptyState() { PkVectorBucketIndexState state = - PkVectorBucketIndexState.fromActivePayloads( - 7, "test-vector-ann", Collections.emptyList()); + new PkVectorBucketIndexState(7, "test-vector-ann", Collections.emptyList()); assertThat(state.vectorFieldId()).isEqualTo(7); assertThat(state.annSegments()).isEmpty(); @@ -85,7 +83,7 @@ private static IndexFileMeta segment( String segmentFileName, String sourceFileName, String indexType) { PrimaryKeyIndexSourceFile sourceFile = new PrimaryKeyIndexSourceFile(sourceFileName, 10); byte[] sourceMeta = - new PrimaryKeyIndexSourceMeta(Collections.singletonList(sourceFile)).serialize(); + new PrimaryKeyIndexSourceMeta(1, Collections.singletonList(sourceFile)).serialize(); return new IndexFileMeta( indexType, segmentFileName, diff --git a/paimon-core/src/test/java/org/apache/paimon/index/pkvector/PrimaryKeyVectorBucketSearchTest.java b/paimon-core/src/test/java/org/apache/paimon/index/pkvector/PrimaryKeyVectorBucketSearchTest.java index 70c9f83953f9..00663b41fd88 100644 --- a/paimon-core/src/test/java/org/apache/paimon/index/pkvector/PrimaryKeyVectorBucketSearchTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/index/pkvector/PrimaryKeyVectorBucketSearchTest.java @@ -447,6 +447,7 @@ private static IndexFileMeta segment(String fileName, List sources long rowCount = sources.stream().mapToLong(DataFileMeta::rowCount).sum(); byte[] sourceMeta = new PrimaryKeyIndexSourceMeta( + 1, sources.stream() .map( source -> diff --git a/paimon-core/src/test/java/org/apache/paimon/operation/PrimaryKeyIndexWriteTest.java b/paimon-core/src/test/java/org/apache/paimon/operation/PrimaryKeyIndexWriteTest.java index 0fd641f15761..4d2110931026 100644 --- a/paimon-core/src/test/java/org/apache/paimon/operation/PrimaryKeyIndexWriteTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/operation/PrimaryKeyIndexWriteTest.java @@ -75,10 +75,6 @@ void testCreatesCoordinatorForBTreeAndBitmapDefinitions() throws Exception { options.put(CoreOptions.DELETION_VECTORS_ENABLED.key(), "true"); options.put(CoreOptions.PK_BTREE_INDEX_COLUMNS.key(), "itemId"); options.put(CoreOptions.PK_BITMAP_INDEX_COLUMNS.key(), "comment"); - options.put("fields.itemId.pk-index.compaction.level-fanout", "7"); - options.put("fields.itemId.pk-index.compaction.stale-ratio-threshold", "0.4"); - options.put("fields.comment.pk-index.compaction.level-fanout", "7"); - options.put("fields.comment.pk-index.compaction.stale-ratio-threshold", "0.4"); TestFileStore store = createStore(options); KeyValueFileStoreWrite write = (KeyValueFileStoreWrite) store.newWrite(); write.withIOManager(ioManager); @@ -90,13 +86,6 @@ void testCreatesCoordinatorForBTreeAndBitmapDefinitions() throws Exception { assertThat(container.primaryKeyIndexMaintainer).isNotNull(); assertThat(container.primaryKeyIndexMaintainer.buildNotCompleted()).isFalse(); - List sortedMaintainers = - (List) readField(container.primaryKeyIndexMaintainer, "sortedMaintainers"); - for (Object sortedMaintainer : sortedMaintainers) { - Object levels = readField(sortedMaintainer, "levels"); - assertThat(readField(levels, "fanout")).isEqualTo(7); - assertThat(readField(levels, "staleRatioThreshold")).isEqualTo(0.4); - } write.close(); } diff --git a/paimon-core/src/test/java/org/apache/paimon/schema/PrimaryKeyFullTextIndexValidationTest.java b/paimon-core/src/test/java/org/apache/paimon/schema/PrimaryKeyFullTextIndexValidationTest.java index f8a59512ac78..39cc905f8b15 100644 --- a/paimon-core/src/test/java/org/apache/paimon/schema/PrimaryKeyFullTextIndexValidationTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/schema/PrimaryKeyFullTextIndexValidationTest.java @@ -165,16 +165,6 @@ void testRejectsColumnConfiguredForAnotherPrimaryKeyIndexFamily() { .hasMessageContaining("at most one primary-key index"); } - @Test - void testRejectsInvalidLsmCompactionOptions() { - Map options = enabledOptions(); - options.put("fields.content.pk-index.compaction.level-fanout", "1"); - - assertThatThrownBy(() -> validateTableSchema(schema(options))) - .hasMessageContaining("fields.content.pk-index.compaction.level-fanout") - .hasMessageContaining("greater than 1"); - } - @Test void testRejectsMalformedIndexOptions() { Map options = enabledOptions(); diff --git a/paimon-core/src/test/java/org/apache/paimon/schema/PrimaryKeySortedIndexValidationTest.java b/paimon-core/src/test/java/org/apache/paimon/schema/PrimaryKeySortedIndexValidationTest.java index 6da0672ada79..d55dd836efc9 100644 --- a/paimon-core/src/test/java/org/apache/paimon/schema/PrimaryKeySortedIndexValidationTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/schema/PrimaryKeySortedIndexValidationTest.java @@ -164,17 +164,6 @@ void testRejectsUnsupportedIndexColumnType() { .hasMessageContaining("not supported by global index"); } - @Test - void testRejectsInvalidCompactionLevelFanout() { - Map options = enabledOptions(); - options.put(CoreOptions.PK_BTREE_INDEX_COLUMNS.key(), "payload"); - options.put("fields.payload.pk-index.compaction.level-fanout", "1"); - - assertThatThrownBy(() -> validateTableSchema(schema(options))) - .hasMessageContaining("fields.payload.pk-index.compaction.level-fanout") - .hasMessageContaining("greater than 1"); - } - private static Map enabledOptions() { Map options = new HashMap<>(); options.put(CoreOptions.BUCKET.key(), "1"); diff --git a/paimon-core/src/test/java/org/apache/paimon/schema/PrimaryKeyVectorIndexValidationTest.java b/paimon-core/src/test/java/org/apache/paimon/schema/PrimaryKeyVectorIndexValidationTest.java index 99136a5099c8..2a666f6ebcd4 100644 --- a/paimon-core/src/test/java/org/apache/paimon/schema/PrimaryKeyVectorIndexValidationTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/schema/PrimaryKeyVectorIndexValidationTest.java @@ -242,26 +242,6 @@ void testRejectsUnsupportedDistanceMetric() { .hasMessageContaining("l2, cosine, inner_product"); } - @Test - void testRejectsInvalidAnnCompactionLevelFanout() { - Map options = enabledOptions(); - options.put("fields.embedding.pk-index.compaction.level-fanout", "1"); - - assertThatThrownBy(() -> validateTableSchema(schema(options))) - .hasMessageContaining("fields.embedding.pk-index.compaction.level-fanout") - .hasMessageContaining("greater than 1"); - } - - @Test - void testRejectsInvalidAnnCompactionStaleRatio() { - Map options = enabledOptions(); - options.put("fields.embedding.pk-index.compaction.stale-ratio-threshold", "1.1"); - - assertThatThrownBy(() -> validateTableSchema(schema(options))) - .hasMessageContaining("fields.embedding.pk-index.compaction.stale-ratio-threshold") - .hasMessageContaining("(0, 1]"); - } - private static Map enabledOptions() { Map options = new HashMap<>(); options.put(CoreOptions.BUCKET.key(), "1"); diff --git a/paimon-core/src/test/java/org/apache/paimon/table/source/FullTextSearchBuilderTest.java b/paimon-core/src/test/java/org/apache/paimon/table/source/FullTextSearchBuilderTest.java index c351f8691a88..5c43f434705a 100644 --- a/paimon-core/src/test/java/org/apache/paimon/table/source/FullTextSearchBuilderTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/table/source/FullTextSearchBuilderTest.java @@ -952,7 +952,7 @@ private void buildAndCommitSourceBackedIndex(FileStoreTable table, String[] docu writer.finish()); byte[] sourceMeta = new PrimaryKeyIndexSourceMeta( - new PrimaryKeyIndexSourceFile("data-file", documents.length)) + 1, new PrimaryKeyIndexSourceFile("data-file", documents.length)) .serialize(); List sourceBackedFiles = new ArrayList<>(); for (IndexFileMeta indexFile : indexFiles) { diff --git a/paimon-core/src/test/java/org/apache/paimon/table/source/PrimaryKeyFullTextReadTest.java b/paimon-core/src/test/java/org/apache/paimon/table/source/PrimaryKeyFullTextReadTest.java index 1be9fe7c1036..c00ecb18a27b 100644 --- a/paimon-core/src/test/java/org/apache/paimon/table/source/PrimaryKeyFullTextReadTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/table/source/PrimaryKeyFullTextReadTest.java @@ -150,7 +150,8 @@ private static DataFileMeta dataFile(String name) { private static IndexFileMeta payload(String source) { byte[] sourceMeta = - new PrimaryKeyIndexSourceMeta(new PrimaryKeyIndexSourceFile(source, 2)).serialize(); + new PrimaryKeyIndexSourceMeta(1, new PrimaryKeyIndexSourceFile(source, 2)) + .serialize(); return new IndexFileMeta( "full-text", "index-" + source, diff --git a/paimon-core/src/test/java/org/apache/paimon/table/source/PrimaryKeyFullTextScanTest.java b/paimon-core/src/test/java/org/apache/paimon/table/source/PrimaryKeyFullTextScanTest.java index ad290b136257..29d843d3ff1f 100644 --- a/paimon-core/src/test/java/org/apache/paimon/table/source/PrimaryKeyFullTextScanTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/table/source/PrimaryKeyFullTextScanTest.java @@ -122,6 +122,24 @@ void testPlansMultiSourceArchiveOnce() { assertThat(split.uncoveredDataFiles()).isEmpty(); } + @Test + void testRejectsPartialLevelCoverage() { + DataFileMeta first = dataFile("data-1", 1, FileSource.COMPACT); + DataFileMeta second = dataFile("data-2", 1, FileSource.COMPACT); + + PrimaryKeyFullTextScan.Plan plan = + PrimaryKeyFullTextScan.plan( + 11, + Collections.singletonList(dataSplit(Arrays.asList(first, second), null)), + Collections.singletonList( + payloadEntry("data-1", FIELD_ID, "partial-level")), + FIELD_ID); + + PrimaryKeyFullTextSearchSplit split = (PrimaryKeyFullTextSearchSplit) plan.splits().get(0); + assertThat(split.payloadFiles()).isEmpty(); + assertThat(split.uncoveredDataFiles()).containsExactly("data-1", "data-2"); + } + @Test @SuppressWarnings({"unchecked", "rawtypes"}) void testCapturesOneSnapshotAndPrunesPartitions() { @@ -216,9 +234,7 @@ private static PrimaryKeyIndexDefinition definition() { FIELD_ID, "full-text", new Options(), - PrimaryKeyIndexDefinition.Family.FULL_TEXT, - 2, - 0.5); + PrimaryKeyIndexDefinition.Family.FULL_TEXT); } private static DataSplit dataSplit( @@ -248,7 +264,7 @@ private static IndexManifestEntry payloadEntry( for (String sourceFile : sourceFiles) { sources.add(new PrimaryKeyIndexSourceFile(sourceFile, 2)); } - byte[] sourceMeta = new PrimaryKeyIndexSourceMeta(sources).serialize(); + byte[] sourceMeta = new PrimaryKeyIndexSourceMeta(1, sources).serialize(); long rowCount = 2L * sources.size(); return new IndexManifestEntry( FileKind.ADD, diff --git a/paimon-core/src/test/java/org/apache/paimon/table/source/PrimaryKeySortedIndexBatchScanTest.java b/paimon-core/src/test/java/org/apache/paimon/table/source/PrimaryKeySortedIndexBatchScanTest.java index 1b6256576d50..f4d8243a3ccc 100644 --- a/paimon-core/src/test/java/org/apache/paimon/table/source/PrimaryKeySortedIndexBatchScanTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/table/source/PrimaryKeySortedIndexBatchScanTest.java @@ -255,26 +255,27 @@ private static DataSplit dataSplit(long snapshotId, DataFileMeta dataFile) { private static DataFileMeta dataFile(String fileName, long rowCount) { return DataFileMeta.forAppend( - fileName, - 100, - rowCount, - SimpleStats.EMPTY_STATS, - 0, - 0, - 1, - Collections.emptyList(), - null, - FileSource.COMPACT, - null, - null, - null, - null); + fileName, + 100, + rowCount, + SimpleStats.EMPTY_STATS, + 0, + 0, + 1, + Collections.emptyList(), + null, + FileSource.COMPACT, + null, + null, + null, + null) + .upgrade(1); } private static IndexFileMeta payload(String fileName, String sourceName, long sourceRowCount) { byte[] sourceMeta = new PrimaryKeyIndexSourceMeta( - new PrimaryKeyIndexSourceFile(sourceName, sourceRowCount)) + 1, new PrimaryKeyIndexSourceFile(sourceName, sourceRowCount)) .serialize(); return new IndexFileMeta( BTreeGlobalIndexerFactory.IDENTIFIER, diff --git a/paimon-core/src/test/java/org/apache/paimon/table/source/PrimaryKeySortedIndexReadTest.java b/paimon-core/src/test/java/org/apache/paimon/table/source/PrimaryKeySortedIndexReadTest.java index 8987fe268d7d..14b476c368b6 100644 --- a/paimon-core/src/test/java/org/apache/paimon/table/source/PrimaryKeySortedIndexReadTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/table/source/PrimaryKeySortedIndexReadTest.java @@ -156,7 +156,6 @@ void testReadAfterIndexCompaction() throws Exception { .option(CoreOptions.DELETION_VECTORS_ENABLED.key(), "true") .option(CoreOptions.TARGET_FILE_SIZE.key(), "1 b") .option(CoreOptions.PK_BTREE_INDEX_COLUMNS.key(), "score") - .option(CoreOptions.primaryKeyIndexCompactionLevelFanoutKey("score"), "2") .build(); catalog.createTable(identifier(), schema, false); FileStoreTable table = getTableDefault(); diff --git a/paimon-core/src/test/java/org/apache/paimon/table/source/PrimaryKeySortedIndexResultTest.java b/paimon-core/src/test/java/org/apache/paimon/table/source/PrimaryKeySortedIndexResultTest.java index 6055e3b500e5..34429e545c0b 100644 --- a/paimon-core/src/test/java/org/apache/paimon/table/source/PrimaryKeySortedIndexResultTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/table/source/PrimaryKeySortedIndexResultTest.java @@ -60,10 +60,10 @@ class PrimaryKeySortedIndexResultTest { @Test void testIndexedEmptyRawAndInvalidFiles() { - DataFileMeta indexed = dataFile("indexed", 5); - DataFileMeta empty = dataFile("empty", 5); - DataFileMeta raw = dataFile("raw", 5); - DataFileMeta invalid = dataFile("invalid", 5); + DataFileMeta indexed = dataFile("indexed", 5, 1); + DataFileMeta empty = dataFile("empty", 5, 2); + DataFileMeta raw = dataFile("raw", 5, 3); + DataFileMeta invalid = dataFile("invalid", 5, 4); List deletionFiles = Arrays.asList( new DeletionFile("dv-indexed", 0, 1, 1L), @@ -77,18 +77,16 @@ void testIndexedEmptyRawAndInvalidFiles() { 7, BTreeGlobalIndexerFactory.IDENTIFIER, new Options(), - PrimaryKeyIndexDefinition.Family.BTREE, - 5, - 0.2); + PrimaryKeyIndexDefinition.Family.BTREE); PrimaryKeySortedIndexScan.Plan plan = PrimaryKeySortedIndexScan.plan( 11, Collections.singletonList(split), Collections.singletonList(definition), Arrays.asList( - payloadEntry(payload("btree-indexed", "indexed", 5)), - payloadEntry(payload("btree-empty", "empty", 5)), - payloadEntry(payload("btree-invalid", "invalid", 5)))); + payloadEntry(payload("btree-indexed", "indexed", 5, 1)), + payloadEntry(payload("btree-empty", "empty", 5, 2)), + payloadEntry(payload("btree-invalid", "invalid", 5, 4)))); RowType rowType = RowType.of(new DataField(7, "f7", DataTypes.INT())); Predicate predicate = new PredicateBuilder(rowType).equal(0, 42); PrimaryKeySortedIndexScan.EvaluatedPlan evaluated = @@ -124,8 +122,8 @@ void testIndexedEmptyRawAndInvalidFiles() { @Test void testNonRawConvertibleSplitPreservesMergeBoundary() { - DataFileMeta first = dataFile("first", 5); - DataFileMeta second = dataFile("second", 5); + DataFileMeta first = dataFile("first", 5, 1); + DataFileMeta second = dataFile("second", 5, 2); List deletionFiles = Arrays.asList( new DeletionFile("dv-first", 0, 1, 1L), @@ -137,17 +135,15 @@ void testNonRawConvertibleSplitPreservesMergeBoundary() { 7, BTreeGlobalIndexerFactory.IDENTIFIER, new Options(), - PrimaryKeyIndexDefinition.Family.BTREE, - 5, - 0.2); + PrimaryKeyIndexDefinition.Family.BTREE); PrimaryKeySortedIndexScan.Plan plan = PrimaryKeySortedIndexScan.plan( 11, Collections.singletonList(split), Collections.singletonList(definition), Arrays.asList( - payloadEntry(payload("btree-first", "first", 5)), - payloadEntry(payload("btree-second", "second", 5)))); + payloadEntry(payload("btree-first", "first", 5, 1)), + payloadEntry(payload("btree-second", "second", 5, 2)))); RowType rowType = RowType.of(new DataField(7, "f7", DataTypes.INT())); Predicate predicate = new PredicateBuilder(rowType).equal(0, 42); PrimaryKeySortedIndexScan.EvaluatedPlan evaluated = @@ -179,9 +175,7 @@ void testFragmentedIndexResultFallsBackToRawSplit() { 7, BTreeGlobalIndexerFactory.IDENTIFIER, new Options(), - PrimaryKeyIndexDefinition.Family.BTREE, - 5, - 0.2); + PrimaryKeyIndexDefinition.Family.BTREE); PrimaryKeySortedIndexScan.Plan plan = PrimaryKeySortedIndexScan.plan( 11, @@ -256,21 +250,26 @@ private static DataSplit dataSplit( } private static DataFileMeta dataFile(String fileName, long rowCount) { + return dataFile(fileName, rowCount, 1); + } + + private static DataFileMeta dataFile(String fileName, long rowCount, int dataLevel) { return DataFileMeta.forAppend( - fileName, - 100, - rowCount, - SimpleStats.EMPTY_STATS, - 0, - 0, - 1, - Collections.emptyList(), - null, - FileSource.COMPACT, - null, - null, - null, - null); + fileName, + 100, + rowCount, + SimpleStats.EMPTY_STATS, + 0, + 0, + 1, + Collections.emptyList(), + null, + FileSource.COMPACT, + null, + null, + null, + null) + .upgrade(dataLevel); } private static IndexManifestEntry payloadEntry(IndexFileMeta payload) { @@ -278,8 +277,14 @@ private static IndexManifestEntry payloadEntry(IndexFileMeta payload) { } private static IndexFileMeta payload(String fileName, String sourceName, long sourceRowCount) { + return payload(fileName, sourceName, sourceRowCount, 1); + } + + private static IndexFileMeta payload( + String fileName, String sourceName, long sourceRowCount, int dataLevel) { byte[] sourceMeta = new PrimaryKeyIndexSourceMeta( + dataLevel, new PrimaryKeyIndexSourceFile(sourceName, sourceRowCount)) .serialize(); return new IndexFileMeta( diff --git a/paimon-core/src/test/java/org/apache/paimon/table/source/PrimaryKeySortedIndexScanTest.java b/paimon-core/src/test/java/org/apache/paimon/table/source/PrimaryKeySortedIndexScanTest.java index e7b72db82e7b..af44371136ac 100644 --- a/paimon-core/src/test/java/org/apache/paimon/table/source/PrimaryKeySortedIndexScanTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/table/source/PrimaryKeySortedIndexScanTest.java @@ -84,12 +84,15 @@ void testPayloadStateIsBuiltOncePerBucketAndDefinition() { 8, BitmapGlobalIndexerFactory.IDENTIFIER, PrimaryKeyIndexDefinition.Family.BITMAP)); - List entries = new java.util.ArrayList<>(); - for (int i = 1; i <= 3; i++) { - String source = "data-" + i; - entries.add(payloadEntry(0, payload("btree-" + i, source, 4, "btree", 7, 4))); - entries.add(payloadEntry(0, payload("bitmap-" + i, source, 4, "bitmap", 8, 4))); - } + List sources = + Arrays.asList( + new PrimaryKeyIndexSourceFile("data-1", 4), + new PrimaryKeyIndexSourceFile("data-2", 4), + new PrimaryKeyIndexSourceFile("data-3", 4)); + List entries = + Arrays.asList( + payloadEntry(0, payload("btree-level", sources, "btree", 7, 12)), + payloadEntry(0, payload("bitmap-level", sources, "bitmap", 8, 12))); try (MockedStatic states = mockStatic( @@ -108,12 +111,12 @@ void testPayloadStateIsBuiltOncePerBucketAndDefinition() { states.verify( times(1), () -> - PkSortedBucketIndexState.fromActivePayloads( + PkSortedBucketIndexState.fromActiveDataFiles( eq(7), eq("btree"), anyList(), anyList())); states.verify( times(1), () -> - PkSortedBucketIndexState.fromActivePayloads( + PkSortedBucketIndexState.fromActiveDataFiles( eq(8), eq("bitmap"), anyList(), anyList())); } } @@ -133,8 +136,7 @@ void testSnapshotScopedGroupPlanning() { PrimaryKeyIndexDefinition.Family.BITMAP)); List entries = Arrays.asList( - payloadEntry(0, payload("btree-0", "data-1", 4, "btree", 7, 2)), - payloadEntry(0, payload("btree-1", "data-1", 4, "btree", 7, 2)), + payloadEntry(0, payload("btree", "data-1", 4, "btree", 7, 4)), payloadEntry(1, payload("wrong-bucket", "data-1", 4, "btree", 7, 4)), payloadEntry(0, payload("wrong-field", "data-1", 4, "btree", 9, 4)), payloadEntry(0, payload("wrong-type", "data-1", 4, "bitmap", 7, 4)), @@ -152,12 +154,12 @@ void testSnapshotScopedGroupPlanning() { assertThat(file.group(7)).isPresent(); assertThat(file.group(7).get().payloads()) .extracting(IndexFileMeta::fileName) - .containsExactly("btree-0", "btree-1"); + .containsExactly("btree"); assertThat(file.group(8)).isEmpty(); } @Test - void testRotatedPayloadsAreUnionedBeforeEvaluation() { + void testDuplicateLevelPayloadsFallBackWithoutCreatingReader() { DataSplit split = dataSplit(11, 0, dataFile("data-1", 4)); PrimaryKeyIndexDefinition definition = definition( @@ -191,16 +193,12 @@ void testRotatedPayloadsAreUnionedBeforeEvaluation() { Collections.singletonList(definition), (ignoredFile, ignoredDefinition, payloads) -> { readersCreated.incrementAndGet(); - assertThat(payloads) - .extracting(IndexFileMeta::fileName) - .containsExactly("btree-0", "btree-1"); return reader; }); - assertThat(readersCreated).hasValue(1); + assertThat(readersCreated).hasValue(0); assertThat(evaluated.files()).hasSize(1); - assertThat(evaluated.files().get(0).result()).isPresent(); - assertThat(evaluated.files().get(0).result().get().results()).containsExactly(3L); + assertThat(evaluated.files().get(0).result()).isEmpty(); } @Test @@ -314,27 +312,33 @@ void testPerFileBooleanFallbackSemantics() { } @Test - void testReaderFailureFallsBackOnlyCurrentFile() { + void testReaderFailureFallsBackForCompleteLevel() { DataSplit split = dataSplit(11, 0, dataFile("data-1", 4), dataFile("data-2", 4)); PrimaryKeyIndexDefinition definition = definition( 7, BTreeGlobalIndexerFactory.IDENTIFIER, PrimaryKeyIndexDefinition.Family.BTREE); + IndexFileMeta mergedPayload = + payload( + "btree-level", + Arrays.asList( + new PrimaryKeyIndexSourceFile("data-1", 4), + new PrimaryKeyIndexSourceFile("data-2", 4)), + "btree", + 7, + 8); PrimaryKeySortedIndexScan.Plan plan = PrimaryKeySortedIndexScan.plan( 11, Collections.singletonList(split), Collections.singletonList(definition), - Arrays.asList( - payloadEntry(0, payload("btree-1", "data-1", 4, "btree", 7, 4)), - payloadEntry(0, payload("btree-2", "data-2", 4, "btree", 7, 4)))); + Collections.singletonList(payloadEntry(0, mergedPayload))); RowType rowType = RowType.of(new DataField(7, "f7", DataTypes.INT())); Predicate predicate = new PredicateBuilder(rowType).equal(0, 42); GlobalIndexReader failedReader = mock(GlobalIndexReader.class); when(failedReader.visitEqual(any(), eq(42))) .thenThrow(new RuntimeException("corrupt index")); - GlobalIndexReader successfulReader = readerWithPositions(1); PrimaryKeySortedIndexScan.EvaluatedPlan evaluated = PrimaryKeySortedIndexScan.evaluate( @@ -342,21 +346,17 @@ void testReaderFailureFallsBackOnlyCurrentFile() { rowType, predicate, Collections.singletonList(definition), - (file, ignoredDefinition, ignoredPayloads) -> - file.dataFile().fileName().equals("data-1") - ? failedReader - : successfulReader); + (file, ignoredDefinition, ignoredPayloads) -> failedReader); assertThat(evaluated.files()).hasSize(2); assertThat(evaluated.files().get(0).result()).isEmpty(); - assertThat(evaluated.files().get(1).result()).isPresent(); - assertThat(evaluated.files().get(1).result().get().results()).containsExactly(1L); + assertThat(evaluated.files().get(1).result()).isEmpty(); } private static PrimaryKeyIndexDefinition definition( int fieldId, String indexType, PrimaryKeyIndexDefinition.Family family) { return new PrimaryKeyIndexDefinition( - "f" + fieldId, fieldId, indexType, new Options(), family, 5, 0.2); + "f" + fieldId, fieldId, indexType, new Options(), family); } private static GlobalIndexReader readerWithPositions(long... rowPositions) { @@ -425,20 +425,21 @@ private static DataSplit dataSplit( private static DataFileMeta dataFile(String fileName, long rowCount) { return DataFileMeta.forAppend( - fileName, - 100, - rowCount, - SimpleStats.EMPTY_STATS, - 0, - 0, - 1, - Collections.emptyList(), - null, - FileSource.COMPACT, - null, - null, - null, - null); + fileName, + 100, + rowCount, + SimpleStats.EMPTY_STATS, + 0, + 0, + 1, + Collections.emptyList(), + null, + FileSource.COMPACT, + null, + null, + null, + null) + .upgrade(1); } private static IndexManifestEntry payloadEntry(int bucket, IndexFileMeta payload) { @@ -467,7 +468,7 @@ private static IndexFileMeta payload( String indexType, int fieldId, long payloadRowCount) { - byte[] sourceMeta = new PrimaryKeyIndexSourceMeta(sourceFiles).serialize(); + byte[] sourceMeta = new PrimaryKeyIndexSourceMeta(1, sourceFiles).serialize(); long sourceRowCount = 0; for (PrimaryKeyIndexSourceFile sourceFile : sourceFiles) { sourceRowCount = Math.addExact(sourceRowCount, sourceFile.rowCount()); diff --git a/paimon-core/src/test/java/org/apache/paimon/table/source/PrimaryKeyVectorScanTest.java b/paimon-core/src/test/java/org/apache/paimon/table/source/PrimaryKeyVectorScanTest.java index 64b1731fc441..1f677eac2849 100644 --- a/paimon-core/src/test/java/org/apache/paimon/table/source/PrimaryKeyVectorScanTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/table/source/PrimaryKeyVectorScanTest.java @@ -237,6 +237,7 @@ private static IndexFileMeta payloadFile() { private static IndexFileMeta payloadFile(String indexType, int fieldId, String fileName) { byte[] sourceMeta = new PrimaryKeyIndexSourceMeta( + 1, Collections.singletonList( new PrimaryKeyIndexSourceFile("data-1", 2))) .serialize(); diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/sink/coordinator/TableWriteCoordinatorTest.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/sink/coordinator/TableWriteCoordinatorTest.java index 4b01a3a1e6f2..87b6bbca6cf7 100644 --- a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/sink/coordinator/TableWriteCoordinatorTest.java +++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/sink/coordinator/TableWriteCoordinatorTest.java @@ -101,7 +101,7 @@ public void testScanVectorIndexPayloads() throws Exception { PrimaryKeyIndexSourceMeta sourceMeta = new PrimaryKeyIndexSourceMeta( - Collections.singletonList(new PrimaryKeyIndexSourceFile("data", 1))); + 1, Collections.singletonList(new PrimaryKeyIndexSourceFile("data", 1))); IndexFileMeta ann = new IndexFileMeta( "test-vector-ann", diff --git a/paimon-full-text/README.md b/paimon-full-text/README.md index 26a08ac99972..2ac1ae0e9b34 100644 --- a/paimon-full-text/README.md +++ b/paimon-full-text/README.md @@ -114,12 +114,10 @@ CREATE TABLE articles ( ); ``` -Paimon creates native archives from complete Level-1-or-higher `COMPACT` data files and -incrementally consolidates them with the shared primary-key index LSM policy. One archive can -cover multiple ordered source files; its row IDs concatenate their physical row positions. The -shared `fields..pk-index.compaction.level-fanout` and -`fields..pk-index.compaction.stale-ratio-threshold` options control size-tier and stale- -source rebuilds. +Paimon creates one native archive for the complete set of eligible `COMPACT` data files in each +Level-1-or-higher data level. Data compaction replaces the affected level archive atomically. One +archive can cover multiple ordered source files; its row IDs concatenate their physical row +positions. Primary-key full-text search currently supports only `global-index.search-mode=fast`. Level-0 and other uncovered files are not searched; their rows become searchable after compaction publishes diff --git a/paimon-full-text/src/test/java/org/apache/paimon/fulltext/index/NativePrimaryKeyFullTextIndexTest.java b/paimon-full-text/src/test/java/org/apache/paimon/fulltext/index/NativePrimaryKeyFullTextIndexTest.java index 567cb2a56e48..85b62c0345f1 100644 --- a/paimon-full-text/src/test/java/org/apache/paimon/fulltext/index/NativePrimaryKeyFullTextIndexTest.java +++ b/paimon-full-text/src/test/java/org/apache/paimon/fulltext/index/NativePrimaryKeyFullTextIndexTest.java @@ -192,7 +192,7 @@ private IndexFileMeta buildArchive(List texts, Options options) th null, result.meta(), new PrimaryKeyIndexSourceMeta( - new PrimaryKeyIndexSourceFile("data-1", texts.size())) + 1, new PrimaryKeyIndexSourceFile("data-1", texts.size())) .serialize()), null); }