From 70927239785a54a6156653a6e3ded0b83d565f42 Mon Sep 17 00:00:00 2001 From: JingsongLi Date: Fri, 10 Jul 2026 23:35:36 +0800 Subject: [PATCH 01/19] [core] Add primary-key vector index options --- .../java/org/apache/paimon/CoreOptions.java | 184 +++++++++++++++ .../PrimaryKeyVectorIndexOptions.java | 106 +++++++++ .../paimon/schema/SchemaValidation.java | 91 ++++++++ .../PrimaryKeyVectorIndexOptionsTest.java | 98 ++++++++ .../PrimaryKeyVectorIndexValidationTest.java | 210 ++++++++++++++++++ 5 files changed, 689 insertions(+) create mode 100644 paimon-core/src/main/java/org/apache/paimon/index/pkvector/PrimaryKeyVectorIndexOptions.java create mode 100644 paimon-core/src/test/java/org/apache/paimon/index/pkvector/PrimaryKeyVectorIndexOptionsTest.java create mode 100644 paimon-core/src/test/java/org/apache/paimon/schema/PrimaryKeyVectorIndexValidationTest.java diff --git a/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java b/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java index b353b5632790..8a8f262bb398 100644 --- a/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java +++ b/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java @@ -2726,6 +2726,98 @@ public String toString() { "The batch size for lateral vector search. Each batch executes vector " + "topK search and table lookup for multiple query vectors."); + /** + * State of the bucket-local vector index for a primary-key table. The definition is persisted + * in table options so every writer uses the same indexed column and algorithm. + */ + public static final ConfigOption PK_VECTOR_INDEX_STATE = + key("pk-vector.index.state") + .enumType(PrimaryKeyVectorIndexState.class) + .defaultValue(PrimaryKeyVectorIndexState.DISABLED) + .withDescription( + "Lifecycle state of the bucket-local primary-key vector index. " + + "Only BUILDING and ACTIVE cause writers to create vector sidecars."); + + public static final ConfigOption PK_VECTOR_INDEX_NAME = + key("pk-vector.index.name") + .stringType() + .noDefaultValue() + .withDescription("Name of the bucket-local primary-key vector index."); + + public static final ConfigOption PK_VECTOR_INDEX_COLUMN = + key("pk-vector.index.column") + .stringType() + .noDefaultValue() + .withDescription("VECTOR column indexed by the primary-key vector index."); + + public static final ConfigOption PK_VECTOR_INDEX_TYPE = + key("pk-vector.index.type") + .stringType() + .noDefaultValue() + .withDescription( + "Vector index algorithm identifier, for example 'ivf-pq'. " + + "The implementation validates it through the vector-index SPI."); + + public static final ConfigOption PK_VECTOR_INDEX_OPTIONS = + key("pk-vector.index.options") + .stringType() + .noDefaultValue() + .withDescription( + "Algorithm-specific options as a JSON object. Unqualified keys are " + + "scoped to pk-vector.index.type; fully qualified index or " + + "fields. keys are preserved."); + + public static final ConfigOption PK_VECTOR_DISTANCE_METRIC = + key("pk-vector.distance.metric") + .stringType() + .defaultValue("inner_product") + .withDescription( + "Distance metric persisted by the primary-key vector index. " + + "Supported values are l2, cosine, and inner_product."); + + public static final ConfigOption PK_VECTOR_L0_MAX_SEGMENTS = + key("pk-vector.l0.max-segments") + .intType() + .defaultValue(8) + .withDescription( + "Maximum raw vector segments in a bucket before vector minor compaction."); + + public static final ConfigOption PK_VECTOR_L0_MAX_ROWS = + key("pk-vector.l0.max-rows") + .longType() + .defaultValue(50_000L) + .withDescription( + "Maximum raw vector rows in a bucket before vector minor compaction."); + + public static final ConfigOption PK_VECTOR_ANN_MIN_ROWS = + key("pk-vector.ann.min-rows") + .longType() + .defaultValue(10_000L) + .withDescription( + "Minimum live rows required before a bucket vector segment is built as ANN."); + + public static final ConfigOption PK_VECTOR_ANN_MAX_ROWS = + key("pk-vector.ann.max-rows") + .longType() + .defaultValue(100_000L) + .withDescription( + "Target maximum rows in one primary-key ANN segment. A single oversized source file is not split."); + + public static final ConfigOption PK_VECTOR_ANN_MAX_SOURCE_FILES = + key("pk-vector.ann.max-source-files") + .intType() + .defaultValue(32) + .withDescription( + "Maximum source data files represented by one primary-key ANN segment."); + + public static final ConfigOption PK_VECTOR_REFINE_FACTOR = + key("pk-vector.refine-factor") + .intType() + .defaultValue(4) + .withDescription( + "Initial ANN candidate multiplier for primary-key vector search. " + + "It is not a visibility correctness boundary."); + @Immutable public static final ConfigOption PK_CLUSTERING_OVERRIDE = key("pk-clustering-override") @@ -4254,6 +4346,68 @@ public int vectorSearchLateralJoinBatchSize() { return options.get(VECTOR_SEARCH_LATERAL_JOIN_BATCH_SIZE); } + public PrimaryKeyVectorIndexState primaryKeyVectorIndexState() { + return options.get(PK_VECTOR_INDEX_STATE); + } + + public boolean primaryKeyVectorIndexEnabled() { + return primaryKeyVectorIndexState() != PrimaryKeyVectorIndexState.DISABLED; + } + + public boolean primaryKeyVectorIndexWriteEnabled() { + PrimaryKeyVectorIndexState state = primaryKeyVectorIndexState(); + return state == PrimaryKeyVectorIndexState.BUILDING + || state == PrimaryKeyVectorIndexState.ACTIVE; + } + + @Nullable + public String primaryKeyVectorIndexName() { + return options.get(PK_VECTOR_INDEX_NAME); + } + + @Nullable + public String primaryKeyVectorIndexColumn() { + return options.get(PK_VECTOR_INDEX_COLUMN); + } + + @Nullable + public String primaryKeyVectorIndexType() { + return options.get(PK_VECTOR_INDEX_TYPE); + } + + @Nullable + public String primaryKeyVectorIndexOptions() { + return options.get(PK_VECTOR_INDEX_OPTIONS); + } + + public String primaryKeyVectorDistanceMetric() { + return options.get(PK_VECTOR_DISTANCE_METRIC).toLowerCase(Locale.ROOT).replace('-', '_'); + } + + public int primaryKeyVectorL0MaxSegments() { + return options.get(PK_VECTOR_L0_MAX_SEGMENTS); + } + + public long primaryKeyVectorL0MaxRows() { + return options.get(PK_VECTOR_L0_MAX_ROWS); + } + + public long primaryKeyVectorAnnMinRows() { + return options.get(PK_VECTOR_ANN_MIN_ROWS); + } + + public long primaryKeyVectorAnnMaxRows() { + return options.get(PK_VECTOR_ANN_MAX_ROWS); + } + + public int primaryKeyVectorAnnMaxSourceFiles() { + return options.get(PK_VECTOR_ANN_MAX_SOURCE_FILES); + } + + public int primaryKeyVectorRefineFactor() { + return options.get(PK_VECTOR_REFINE_FACTOR); + } + /** Specifies the merge engine for table with primary key. */ public enum MergeEngine implements DescribedEnum { DEDUPLICATE("deduplicate", "De-duplicate and keep the last row."), @@ -5082,6 +5236,36 @@ public InlineElement getDescription() { } } + /** Lifecycle state of a bucket-local primary-key vector index. */ + public enum PrimaryKeyVectorIndexState implements DescribedEnum { + DISABLED("disabled", "No primary-key vector index is configured."), + BUILDING("building", "Existing files are being backfilled; new files create raw sidecars."), + ACTIVE( + "active", + "The index is available for vector search and new files create raw sidecars."), + DROPPING( + "dropping", + "New sidecars are disabled while existing vector index files are removed."); + + private final String value; + private final String description; + + PrimaryKeyVectorIndexState(String value, String description) { + this.value = value; + this.description = description; + } + + @Override + public String toString() { + return value; + } + + @Override + public InlineElement getDescription() { + return text(description); + } + } + /** Strategy for handling rows whose nested-key contains null values. */ public enum NestedKeyNullStrategy implements DescribedEnum { MERGE( diff --git a/paimon-core/src/main/java/org/apache/paimon/index/pkvector/PrimaryKeyVectorIndexOptions.java b/paimon-core/src/main/java/org/apache/paimon/index/pkvector/PrimaryKeyVectorIndexOptions.java new file mode 100644 index 000000000000..0cfc1a911cf9 --- /dev/null +++ b/paimon-core/src/main/java/org/apache/paimon/index/pkvector/PrimaryKeyVectorIndexOptions.java @@ -0,0 +1,106 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.index.pkvector; + +import org.apache.paimon.CoreOptions; +import org.apache.paimon.options.Options; +import org.apache.paimon.utils.JsonSerdeUtil; + +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.TreeMap; + +import static org.apache.paimon.utils.Preconditions.checkArgument; + +/** Resolves and fingerprints algorithm options for the primary-key vector index. */ +public final class PrimaryKeyVectorIndexOptions { + + private PrimaryKeyVectorIndexOptions() {} + + public static Options resolve(CoreOptions coreOptions) { + Options resolved = new Options(coreOptions.toConfiguration().toMap()); + for (Map.Entry option : algorithmOptions(coreOptions).entrySet()) { + resolved.setString(option.getKey(), option.getValue()); + } + return resolved; + } + + public static byte[] hash(CoreOptions coreOptions) { + String canonicalJson = JsonSerdeUtil.toJson(algorithmOptions(coreOptions)); + try { + return MessageDigest.getInstance("SHA-256") + .digest(canonicalJson.getBytes(StandardCharsets.UTF_8)); + } catch (NoSuchAlgorithmException e) { + throw new IllegalStateException("SHA-256 is not available.", e); + } + } + + private static Map algorithmOptions(CoreOptions coreOptions) { + String algorithm = coreOptions.primaryKeyVectorIndexType(); + checkArgument( + algorithm != null && !algorithm.trim().isEmpty(), + "pk-vector.index.type must be configured before resolving index options."); + TreeMap options = new TreeMap<>(); + String field = coreOptions.primaryKeyVectorIndexColumn(); + String algorithmPrefix = algorithm + "."; + String fieldPrefix = field == null ? null : "fields." + field + "."; + for (Map.Entry entry : coreOptions.toConfiguration().toMap().entrySet()) { + if (entry.getKey().startsWith(algorithmPrefix) + || (fieldPrefix != null && entry.getKey().startsWith(fieldPrefix))) { + options.put(entry.getKey(), entry.getValue()); + } + } + String serialized = coreOptions.primaryKeyVectorIndexOptions(); + if (serialized != null && !serialized.trim().isEmpty()) { + LinkedHashMap parsed; + try { + parsed = JsonSerdeUtil.parseJsonMap(serialized, String.class); + } catch (RuntimeException e) { + throw new IllegalArgumentException( + "pk-vector.index.options must be a JSON object of option key-value pairs.", + e); + } + for (Map.Entry entry : parsed.entrySet()) { + String key = entry.getKey(); + String value = entry.getValue(); + checkArgument( + key != null && !key.trim().isEmpty(), + "pk-vector.index.options contains an empty option key."); + checkArgument( + value != null, + "pk-vector.index.options value for key %s must not be null.", + key); + String qualifiedKey = + key.startsWith(algorithmPrefix) || key.startsWith("fields.") + ? key + : algorithmPrefix + key; + String previous = options.put(qualifiedKey, value); + checkArgument( + previous == null || previous.equals(value), + "pk-vector.index.options defines conflicting values for %s.", + qualifiedKey); + } + } + options.put(algorithmPrefix + "metric", coreOptions.primaryKeyVectorDistanceMetric()); + return options; + } +} diff --git a/paimon-core/src/main/java/org/apache/paimon/schema/SchemaValidation.java b/paimon-core/src/main/java/org/apache/paimon/schema/SchemaValidation.java index 56b4676dcab5..0b5c930e9996 100644 --- a/paimon-core/src/main/java/org/apache/paimon/schema/SchemaValidation.java +++ b/paimon-core/src/main/java/org/apache/paimon/schema/SchemaValidation.java @@ -29,6 +29,7 @@ import org.apache.paimon.fileindex.FileIndexerFactory; import org.apache.paimon.fileindex.FileIndexerFactoryUtils; import org.apache.paimon.format.FileFormat; +import org.apache.paimon.index.pkvector.PrimaryKeyVectorIndexOptions; import org.apache.paimon.mergetree.compact.aggregate.FieldAggregator; import org.apache.paimon.mergetree.compact.aggregate.factory.FieldAggregatorFactory; import org.apache.paimon.options.ConfigOption; @@ -350,6 +351,8 @@ public static void validateTableSchema(TableSchema schema, Set dynamicOp fieldNamesSpecifiedAsVector.isEmpty(), "Some of the columns specified as vector-field are unknown."); + validatePrimaryKeyVectorIndex(schema, options); + validateMergeFunctionFactory(schema); validateMapStorageLayout(schema, options); @@ -883,6 +886,94 @@ private static void validateForDeletionVectors(CoreOptions options) { } } + private static void validatePrimaryKeyVectorIndex(TableSchema schema, CoreOptions options) { + if (!options.primaryKeyVectorIndexEnabled()) { + return; + } + + String indexName = options.primaryKeyVectorIndexName(); + String indexColumn = options.primaryKeyVectorIndexColumn(); + String indexType = options.primaryKeyVectorIndexType(); + checkArgument( + !StringUtils.isNullOrWhitespaceOnly(indexName), + "pk-vector.index.name must be configured when pk-vector.index.state = %s.", + options.primaryKeyVectorIndexState()); + checkArgument( + !StringUtils.isNullOrWhitespaceOnly(indexColumn), + "pk-vector.index.column must be configured when pk-vector.index.state = %s.", + options.primaryKeyVectorIndexState()); + checkArgument( + !StringUtils.isNullOrWhitespaceOnly(indexType), + "pk-vector.index.type must be configured when pk-vector.index.state = %s.", + options.primaryKeyVectorIndexState()); + checkArgument( + !schema.primaryKeys().isEmpty(), + "pk-vector.index.state = %s requires a primary-key table.", + options.primaryKeyVectorIndexState()); + checkArgument( + options.deletionVectorsEnabled(), + "pk-vector.index.state = %s requires deletion-vectors.enabled = true.", + options.primaryKeyVectorIndexState()); + checkArgument( + options.mergeEngine() == MergeEngine.DEDUPLICATE + || options.mergeEngine() == MergeEngine.PARTIAL_UPDATE, + "Primary-key vector index only supports merge-engine = deduplicate or partial-update, but is %s.", + options.mergeEngine()); + checkArgument( + !options.deletionVectorsMergeOnRead(), + "Primary-key vector index with merge-engine = %s requires deletion-vectors.merge-on-read = false.", + options.mergeEngine()); + checkArgument( + options.bucket() > 0, + "Primary-key vector index requires fixed bucket mode (bucket > 0), but bucket is %s.", + options.bucket()); + checkArgument( + !options.pkClusteringOverride(), + "Primary-key vector index does not support pk-clustering-override."); + PrimaryKeyVectorIndexOptions.resolve(options); + + DataField vectorField = + schema.fields().stream() + .filter(field -> field.name().equals(indexColumn)) + .findFirst() + .orElse(null); + checkArgument( + vectorField != null && vectorField.type().getTypeRoot() == VECTOR, + "pk-vector.index.column '%s' must reference a VECTOR column.", + indexColumn); + checkArgument( + ((VectorType) vectorField.type()).getElementType().getTypeRoot() + == DataTypeRoot.FLOAT, + "pk-vector.index.column '%s' must use FLOAT elements.", + indexColumn); + checkArgument( + Arrays.asList("l2", "cosine", "inner_product") + .contains(options.primaryKeyVectorDistanceMetric()), + "pk-vector.distance.metric must be one of l2, cosine, inner_product, but is %s.", + options.primaryKeyVectorDistanceMetric()); + checkArgument( + options.primaryKeyVectorL0MaxSegments() > 0, + "pk-vector.l0.max-segments must be greater than 0."); + checkArgument( + options.primaryKeyVectorL0MaxRows() > 0, + "pk-vector.l0.max-rows must be greater than 0."); + checkArgument( + options.primaryKeyVectorAnnMinRows() > 0, + "pk-vector.ann.min-rows must be greater than 0."); + checkArgument( + options.primaryKeyVectorAnnMaxRows() > 0, + "pk-vector.ann.max-rows must be greater than 0."); + checkArgument( + options.primaryKeyVectorAnnMaxRows() >= options.primaryKeyVectorAnnMinRows(), + "pk-vector.ann.max-rows must be greater than or equal to pk-vector.ann.min-rows."); + checkArgument( + options.primaryKeyVectorAnnMaxSourceFiles() > 0, + "pk-vector.ann.max-source-files must be greater than 0."); + checkArgument( + options.primaryKeyVectorRefineFactor() > 0, + "pk-vector.refine-factor must be greater than 0."); + } + private static void validateSequenceField(TableSchema schema, CoreOptions options) { List sequenceField = options.sequenceField(); if (!sequenceField.isEmpty()) { diff --git a/paimon-core/src/test/java/org/apache/paimon/index/pkvector/PrimaryKeyVectorIndexOptionsTest.java b/paimon-core/src/test/java/org/apache/paimon/index/pkvector/PrimaryKeyVectorIndexOptionsTest.java new file mode 100644 index 000000000000..c89141aa1521 --- /dev/null +++ b/paimon-core/src/test/java/org/apache/paimon/index/pkvector/PrimaryKeyVectorIndexOptionsTest.java @@ -0,0 +1,98 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.index.pkvector; + +import org.apache.paimon.CoreOptions; +import org.apache.paimon.options.Options; + +import org.junit.jupiter.api.Test; + +import java.util.HashMap; +import java.util.Map; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/** Tests for {@link PrimaryKeyVectorIndexOptions}. */ +class PrimaryKeyVectorIndexOptionsTest { + + @Test + void testResolvesShortAndQualifiedAlgorithmOptions() { + CoreOptions coreOptions = + coreOptions( + "{\"nlist\":64,\"ivf-pq.pq.m\":\"8\"," + "\"fields.embedding.hnsw.m\":16}"); + + Options resolved = PrimaryKeyVectorIndexOptions.resolve(coreOptions); + + assertThat(resolved.get("ivf-pq.nlist")).isEqualTo("64"); + assertThat(resolved.get("ivf-pq.pq.m")).isEqualTo("8"); + assertThat(resolved.get("fields.embedding.hnsw.m")).isEqualTo("16"); + assertThat(resolved.get("ivf-pq.metric")).isEqualTo("l2"); + } + + @Test + void testHashIsCanonicalAcrossJsonPropertyOrder() { + assertThat(PrimaryKeyVectorIndexOptions.hash(coreOptions("{\"nlist\":64,\"pq.m\":8}"))) + .containsExactly( + PrimaryKeyVectorIndexOptions.hash( + coreOptions("{\"pq.m\":\"8\",\"nlist\":\"64\"}"))); + } + + @Test + void testHashIncludesEffectiveTopLevelAlgorithmOptions() { + assertThat(PrimaryKeyVectorIndexOptions.hash(coreOptions(null, "ivf-pq.nlist", "64"))) + .isNotEqualTo( + PrimaryKeyVectorIndexOptions.hash(coreOptions(null, "ivf-pq.nlist", "65"))); + } + + @Test + void testRejectsNonObjectOptions() { + assertThatThrownBy(() -> PrimaryKeyVectorIndexOptions.resolve(coreOptions("[1,2]"))) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("pk-vector.index.options") + .hasMessageContaining("JSON object"); + } + + @Test + void testAnnBuildBoundsDefaults() { + CoreOptions options = coreOptions(null); + + assertThat(options.primaryKeyVectorAnnMaxRows()).isEqualTo(100_000L); + assertThat(options.primaryKeyVectorAnnMaxSourceFiles()).isEqualTo(32); + } + + private static CoreOptions coreOptions(String indexOptions) { + return coreOptions(indexOptions, null, null); + } + + private static CoreOptions coreOptions( + String indexOptions, String additionalKey, String additionalValue) { + Map options = new HashMap<>(); + options.put(CoreOptions.PK_VECTOR_INDEX_TYPE.key(), "ivf-pq"); + options.put(CoreOptions.PK_VECTOR_INDEX_COLUMN.key(), "embedding"); + options.put(CoreOptions.PK_VECTOR_DISTANCE_METRIC.key(), "l2"); + if (indexOptions != null) { + options.put(CoreOptions.PK_VECTOR_INDEX_OPTIONS.key(), indexOptions); + } + if (additionalKey != null) { + options.put(additionalKey, additionalValue); + } + return new CoreOptions(options); + } +} diff --git a/paimon-core/src/test/java/org/apache/paimon/schema/PrimaryKeyVectorIndexValidationTest.java b/paimon-core/src/test/java/org/apache/paimon/schema/PrimaryKeyVectorIndexValidationTest.java new file mode 100644 index 000000000000..f8277792f14e --- /dev/null +++ b/paimon-core/src/test/java/org/apache/paimon/schema/PrimaryKeyVectorIndexValidationTest.java @@ -0,0 +1,210 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.schema; + +import org.apache.paimon.CoreOptions; +import org.apache.paimon.types.DataField; +import org.apache.paimon.types.DataTypes; + +import org.junit.jupiter.api.Test; + +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +import static org.apache.paimon.schema.SchemaValidation.validateTableSchema; +import static org.assertj.core.api.Assertions.assertThatCode; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/** Tests for primary-key vector index option validation. */ +class PrimaryKeyVectorIndexValidationTest { + + @Test + void testValidPrimaryKeyVectorIndex() { + assertThatCode(() -> validateTableSchema(schema(enabledOptions()))) + .doesNotThrowAnyException(); + } + + @Test + void testRequiresPrimaryKeyTable() { + Map options = enabledOptions(); + options.put(CoreOptions.BUCKET_KEY.key(), "id"); + assertThatThrownBy( + () -> + validateTableSchema( + new TableSchema( + 0, + fields(), + 0, + Collections.emptyList(), + Collections.emptyList(), + options, + ""))) + .hasMessageContaining("requires a primary-key table"); + } + + @Test + void testRequiresDeletionVectors() { + Map options = enabledOptions(); + options.put(CoreOptions.DELETION_VECTORS_ENABLED.key(), "false"); + + assertThatThrownBy(() -> validateTableSchema(schema(options))) + .hasMessageContaining("requires deletion-vectors.enabled = true"); + } + + @Test + void testSupportsPartialUpdateMergeEngine() { + Map options = enabledOptions(); + options.put(CoreOptions.MERGE_ENGINE.key(), "partial-update"); + + assertThatCode(() -> validateTableSchema(schema(options))).doesNotThrowAnyException(); + } + + @Test + void testPartialUpdateRejectsDeletionVectorMergeOnRead() { + Map options = enabledOptions(); + options.put(CoreOptions.MERGE_ENGINE.key(), "partial-update"); + options.put(CoreOptions.DELETION_VECTORS_MERGE_ON_READ.key(), "true"); + + assertThatThrownBy(() -> validateTableSchema(schema(options))) + .hasMessageContaining( + "partial-update requires deletion-vectors.merge-on-read = false"); + } + + @Test + void testDeduplicateRejectsDeletionVectorMergeOnRead() { + Map options = enabledOptions(); + options.put(CoreOptions.DELETION_VECTORS_MERGE_ON_READ.key(), "true"); + + assertThatThrownBy(() -> validateTableSchema(schema(options))) + .hasMessageContaining("requires deletion-vectors.merge-on-read = false"); + } + + @Test + void testRequiresFixedBucket() { + Map options = enabledOptions(); + options.put(CoreOptions.BUCKET.key(), "-1"); + + assertThatThrownBy(() -> validateTableSchema(schema(options))) + .hasMessageContaining("requires fixed bucket mode"); + } + + @Test + void testRejectsPkClusteringOverride() { + Map options = enabledOptions(); + options.put(CoreOptions.PK_CLUSTERING_OVERRIDE.key(), "true"); + options.put(CoreOptions.CLUSTERING_COLUMNS.key(), "payload"); + + assertThatThrownBy(() -> validateTableSchema(schema(options))) + .hasMessageContaining("does not support pk-clustering-override"); + } + + @Test + void testRequiresVectorColumn() { + Map options = enabledOptions(); + options.put(CoreOptions.PK_VECTOR_INDEX_COLUMN.key(), "payload"); + + assertThatThrownBy(() -> validateTableSchema(schema(options))) + .hasMessageContaining("must reference a VECTOR column"); + } + + @Test + void testRequiresCompleteDefinitionOutsideDisabledState() { + Map options = enabledOptions(); + options.remove(CoreOptions.PK_VECTOR_INDEX_NAME.key()); + + assertThatThrownBy(() -> validateTableSchema(schema(options))) + .hasMessageContaining("pk-vector.index.name must be configured"); + } + + @Test + void testRequiresFloatVectorElements() { + TableSchema schema = + new TableSchema( + 0, + Arrays.asList( + new DataField(0, "id", DataTypes.INT().notNull()), + new DataField( + 1, "embedding", DataTypes.VECTOR(8, DataTypes.DOUBLE()))), + 1, + Collections.emptyList(), + Collections.singletonList("id"), + enabledOptions(), + ""); + + assertThatThrownBy(() -> validateTableSchema(schema)) + .hasMessageContaining("must use FLOAT elements"); + } + + @Test + void testRejectsUnsupportedDistanceMetric() { + Map options = enabledOptions(); + options.put(CoreOptions.PK_VECTOR_DISTANCE_METRIC.key(), "manhattan"); + + assertThatThrownBy(() -> validateTableSchema(schema(options))) + .hasMessageContaining("pk-vector.distance.metric") + .hasMessageContaining("l2, cosine, inner_product"); + } + + @Test + void testRejectsInvalidAnnBuildBounds() { + Map invalidRowOptions = enabledOptions(); + invalidRowOptions.put(CoreOptions.PK_VECTOR_ANN_MIN_ROWS.key(), "100"); + invalidRowOptions.put(CoreOptions.PK_VECTOR_ANN_MAX_ROWS.key(), "99"); + assertThatThrownBy(() -> validateTableSchema(schema(invalidRowOptions))) + .hasMessageContaining("pk-vector.ann.max-rows") + .hasMessageContaining("greater than or equal"); + + Map invalidSourceFileOptions = enabledOptions(); + invalidSourceFileOptions.put(CoreOptions.PK_VECTOR_ANN_MAX_SOURCE_FILES.key(), "0"); + assertThatThrownBy(() -> validateTableSchema(schema(invalidSourceFileOptions))) + .hasMessageContaining("pk-vector.ann.max-source-files") + .hasMessageContaining("greater than 0"); + } + + private static Map enabledOptions() { + Map options = new HashMap<>(); + options.put(CoreOptions.BUCKET.key(), "1"); + options.put(CoreOptions.DELETION_VECTORS_ENABLED.key(), "true"); + options.put(CoreOptions.PK_VECTOR_INDEX_STATE.key(), "building"); + options.put(CoreOptions.PK_VECTOR_INDEX_NAME.key(), "embedding_index"); + options.put(CoreOptions.PK_VECTOR_INDEX_COLUMN.key(), "embedding"); + options.put(CoreOptions.PK_VECTOR_INDEX_TYPE.key(), "ivf-pq"); + return options; + } + + private static TableSchema schema(Map options) { + return new TableSchema( + 0, + fields(), + 0, + Collections.emptyList(), + Collections.singletonList("id"), + options, + ""); + } + + private static java.util.List fields() { + return Arrays.asList( + new DataField(0, "id", DataTypes.INT().notNull()), + new DataField(1, "embedding", DataTypes.VECTOR(8, DataTypes.FLOAT())), + new DataField(2, "payload", DataTypes.STRING())); + } +} From e278e16aa57d2c5db382b3f288e890bbae1a54a8 Mon Sep 17 00:00:00 2001 From: JingsongLi Date: Fri, 10 Jul 2026 23:37:38 +0800 Subject: [PATCH 02/19] [core] Add immutable primary-key vector payloads --- .../testvector/TestVectorGlobalIndexer.java | 19 +- .../TestVectorGlobalIndexerTest.java | 41 ++ .../pkvector/PkVectorAnnSegmentFile.java | 373 ++++++++++++++++++ .../pkvector/PkVectorAnnSegmentSearcher.java | 291 ++++++++++++++ .../pkvector/PkVectorRawSegmentFile.java | 208 ++++++++++ .../index/pkvector/PkVectorSegmentMeta.java | 334 ++++++++++++++++ .../pkvector/RawVectorSidecarReader.java | 247 ++++++++++++ .../pkvector/RawVectorSidecarWriter.java | 163 ++++++++ .../io/KeyValueVectorSidecarWriter.java | 42 ++ .../pkvector/PkVectorAnnSegmentFileTest.java | 320 +++++++++++++++ .../pkvector/PkVectorSegmentMetaTest.java | 116 ++++++ .../index/pkvector/RawVectorSidecarTest.java | 166 ++++++++ 12 files changed, 2316 insertions(+), 4 deletions(-) create mode 100644 paimon-common/src/test/java/org/apache/paimon/globalindex/testvector/TestVectorGlobalIndexerTest.java create mode 100644 paimon-core/src/main/java/org/apache/paimon/index/pkvector/PkVectorAnnSegmentFile.java create mode 100644 paimon-core/src/main/java/org/apache/paimon/index/pkvector/PkVectorAnnSegmentSearcher.java create mode 100644 paimon-core/src/main/java/org/apache/paimon/index/pkvector/PkVectorRawSegmentFile.java create mode 100644 paimon-core/src/main/java/org/apache/paimon/index/pkvector/PkVectorSegmentMeta.java create mode 100644 paimon-core/src/main/java/org/apache/paimon/index/pkvector/RawVectorSidecarReader.java create mode 100644 paimon-core/src/main/java/org/apache/paimon/index/pkvector/RawVectorSidecarWriter.java create mode 100644 paimon-core/src/main/java/org/apache/paimon/io/KeyValueVectorSidecarWriter.java create mode 100644 paimon-core/src/test/java/org/apache/paimon/index/pkvector/PkVectorAnnSegmentFileTest.java create mode 100644 paimon-core/src/test/java/org/apache/paimon/index/pkvector/PkVectorSegmentMetaTest.java create mode 100644 paimon-core/src/test/java/org/apache/paimon/index/pkvector/RawVectorSidecarTest.java diff --git a/paimon-common/src/test/java/org/apache/paimon/globalindex/testvector/TestVectorGlobalIndexer.java b/paimon-common/src/test/java/org/apache/paimon/globalindex/testvector/TestVectorGlobalIndexer.java index 0b6d8d9fd63f..eb0365427af2 100644 --- a/paimon-common/src/test/java/org/apache/paimon/globalindex/testvector/TestVectorGlobalIndexer.java +++ b/paimon-common/src/test/java/org/apache/paimon/globalindex/testvector/TestVectorGlobalIndexer.java @@ -28,6 +28,7 @@ import org.apache.paimon.types.ArrayType; import org.apache.paimon.types.DataType; import org.apache.paimon.types.FloatType; +import org.apache.paimon.types.VectorType; import java.io.IOException; import java.util.List; @@ -74,17 +75,27 @@ public class TestVectorGlobalIndexer implements VectorGlobalIndexer { public TestVectorGlobalIndexer(DataType fieldType, Options options) { checkArgument( - fieldType instanceof ArrayType - && ((ArrayType) fieldType).getElementType() instanceof FloatType, - "TestVectorGlobalIndexer only supports ARRAY, but got: " + fieldType); + isFloatVector(fieldType), + "TestVectorGlobalIndexer only supports VECTOR or ARRAY, but got: " + + fieldType); this.fieldType = fieldType; - this.dimension = options.getInteger(OPT_DIMENSION, 0); + this.dimension = + fieldType instanceof VectorType + ? ((VectorType) fieldType).getLength() + : options.getInteger(OPT_DIMENSION, 0); this.metric = options.getString(OPT_METRIC, "l2"); this.reverseScore = options.getBoolean(OPT_REVERSE_SCORE, false); this.requiredOptionKey = options.getString(OPT_REQUIRED_OPTION_KEY, null); this.requiredOptionValue = options.getString(OPT_REQUIRED_OPTION_VALUE, null); } + private static boolean isFloatVector(DataType fieldType) { + return (fieldType instanceof VectorType + && ((VectorType) fieldType).getElementType() instanceof FloatType) + || (fieldType instanceof ArrayType + && ((ArrayType) fieldType).getElementType() instanceof FloatType); + } + @Override public GlobalIndexWriter createWriter(GlobalIndexFileWriter fileWriter) throws IOException { return new TestVectorGlobalIndexWriter(fileWriter, dimension); diff --git a/paimon-common/src/test/java/org/apache/paimon/globalindex/testvector/TestVectorGlobalIndexerTest.java b/paimon-common/src/test/java/org/apache/paimon/globalindex/testvector/TestVectorGlobalIndexerTest.java new file mode 100644 index 000000000000..7ab5c45cefd4 --- /dev/null +++ b/paimon-common/src/test/java/org/apache/paimon/globalindex/testvector/TestVectorGlobalIndexerTest.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.globalindex.testvector; + +import org.apache.paimon.options.Options; +import org.apache.paimon.types.DataTypes; + +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.Assertions.assertThat; + +/** Tests for {@link TestVectorGlobalIndexer}. */ +class TestVectorGlobalIndexerTest { + + @Test + void testUsesDimensionFromVectorType() { + Options options = new Options(); + options.setString(TestVectorGlobalIndexer.OPT_DIMENSION, "99"); + + TestVectorGlobalIndexer indexer = + new TestVectorGlobalIndexer(DataTypes.VECTOR(2, DataTypes.FLOAT()), options); + + assertThat(indexer.dimension()).isEqualTo(2); + } +} diff --git a/paimon-core/src/main/java/org/apache/paimon/index/pkvector/PkVectorAnnSegmentFile.java b/paimon-core/src/main/java/org/apache/paimon/index/pkvector/PkVectorAnnSegmentFile.java new file mode 100644 index 000000000000..5c07c5dc035e --- /dev/null +++ b/paimon-core/src/main/java/org/apache/paimon/index/pkvector/PkVectorAnnSegmentFile.java @@ -0,0 +1,373 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.index.pkvector; + +import org.apache.paimon.fs.FileIO; +import org.apache.paimon.fs.Path; +import org.apache.paimon.fs.PositionOutputStream; +import org.apache.paimon.globalindex.GlobalIndexSingleColumnWriter; +import org.apache.paimon.globalindex.GlobalIndexWriter; +import org.apache.paimon.globalindex.GlobalIndexer; +import org.apache.paimon.globalindex.ResultEntry; +import org.apache.paimon.globalindex.VectorGlobalIndexer; +import org.apache.paimon.globalindex.io.GlobalIndexFileWriter; +import org.apache.paimon.index.GlobalIndexMeta; +import org.apache.paimon.index.IndexFile; +import org.apache.paimon.index.IndexFileMeta; +import org.apache.paimon.index.IndexPathFactory; +import org.apache.paimon.io.DataFileMeta; +import org.apache.paimon.options.Options; +import org.apache.paimon.types.DataField; +import org.apache.paimon.types.VectorType; +import org.apache.paimon.utils.IOUtils; + +import javax.annotation.Nullable; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.function.LongPredicate; + +import static org.apache.paimon.index.pkvector.PkVectorSegmentMeta.OrdinalLayout.FILE_POSITION; +import static org.apache.paimon.index.pkvector.PkVectorSegmentMeta.OrdinalLayout.ROW_POSITION; +import static org.apache.paimon.index.pkvector.PkVectorSegmentMeta.Role.ANN; +import static org.apache.paimon.utils.Preconditions.checkArgument; + +/** Builds immutable ANN payloads whose index ids are source data-file row positions. */ +public class PkVectorAnnSegmentFile extends IndexFile { + + public static final String PK_VECTOR_ANN = "pk-vector-ann"; + + public PkVectorAnnSegmentFile(FileIO fileIO, IndexPathFactory pathFactory) { + super(fileIO, pathFactory); + } + + public IndexFileMeta buildSingleSource( + DataFileMeta sourceFile, + RawVectorSidecarReader rawVectors, + DataField vectorField, + Options indexOptions, + String indexDefinitionId, + String vectorTypeFingerprint, + String metric, + String algorithm, + byte[] optionsHash, + long buildSnapshotId, + LongPredicate excludedPosition) + throws IOException { + return build( + Collections.singletonList(new Source(sourceFile, rawVectors, excludedPosition)), + ROW_POSITION, + vectorField, + indexOptions, + indexDefinitionId, + vectorTypeFingerprint, + metric, + algorithm, + optionsHash, + buildSnapshotId); + } + + public IndexFileMeta buildMultiSource( + List sources, + DataField vectorField, + Options indexOptions, + String indexDefinitionId, + String vectorTypeFingerprint, + String metric, + String algorithm, + byte[] optionsHash, + long buildSnapshotId) + throws IOException { + checkArgument( + sources.size() > 1, + "A multi-source ANN segment must reference at least two source files."); + return build( + sources, + FILE_POSITION, + vectorField, + indexOptions, + indexDefinitionId, + vectorTypeFingerprint, + metric, + algorithm, + optionsHash, + buildSnapshotId); + } + + IndexFileMeta buildSources( + List sources, + DataField vectorField, + Options indexOptions, + String indexDefinitionId, + String vectorTypeFingerprint, + String metric, + String algorithm, + byte[] optionsHash, + long buildSnapshotId) + throws IOException { + return build( + sources, + sources.size() == 1 ? ROW_POSITION : FILE_POSITION, + vectorField, + indexOptions, + indexDefinitionId, + vectorTypeFingerprint, + metric, + algorithm, + optionsHash, + buildSnapshotId); + } + + private IndexFileMeta build( + List sources, + PkVectorSegmentMeta.OrdinalLayout ordinalLayout, + DataField vectorField, + Options indexOptions, + String indexDefinitionId, + String vectorTypeFingerprint, + String metric, + String algorithm, + byte[] optionsHash, + long buildSnapshotId) + throws IOException { + checkArgument(!sources.isEmpty(), "An ANN segment must reference source files."); + long totalRowCount = 0; + List sourceFiles = new ArrayList<>(sources.size()); + for (Source source : sources) { + totalRowCount = Math.addExact(totalRowCount, source.sourceFile.rowCount()); + sourceFiles.add(source.sourceFile); + } + + GlobalIndexer indexer = GlobalIndexer.create(algorithm, vectorField, indexOptions); + checkArgument( + indexer instanceof VectorGlobalIndexer, + "Index algorithm %s does not implement VectorGlobalIndexer.", + algorithm); + String indexerMetric = normalizeMetric(((VectorGlobalIndexer) indexer).metric()); + checkArgument( + normalizeMetric(metric).equals(indexerMetric), + "Configured metric %s does not match index algorithm metric %s.", + metric, + indexerMetric); + + SegmentFileWriter fileWriter = new SegmentFileWriter(); + GlobalIndexWriter writer = null; + boolean success = false; + try { + writer = indexer.createWriter(fileWriter); + checkArgument( + writer instanceof GlobalIndexSingleColumnWriter, + "Index algorithm %s does not create a single-column writer.", + algorithm); + GlobalIndexSingleColumnWriter singleColumnWriter = + (GlobalIndexSingleColumnWriter) writer; + long liveRowCount = 0; + long fileOffset = 0; + int dimension = -1; + for (Source source : sources) { + RawVectorSidecarReader rawVectors = source.openReader(); + try { + checkArgument( + rawVectors.rowCount() == source.sourceFile.rowCount(), + "Raw vector row count %s does not match source file %s row count %s.", + rawVectors.rowCount(), + source.sourceFile.fileName(), + source.sourceFile.rowCount()); + if (dimension < 0) { + dimension = rawVectors.dimension(); + } + checkArgument( + rawVectors.dimension() == dimension, + "Raw vector source %s dimension %s does not match dimension %s.", + source.sourceFile.fileName(), + rawVectors.dimension(), + dimension); + if (vectorField.type() instanceof VectorType) { + checkArgument( + ((VectorType) vectorField.type()).getLength() == dimension, + "Vector field dimension %s does not match raw vector dimension %s.", + ((VectorType) vectorField.type()).getLength(), + dimension); + } + + float[] vector = new float[dimension]; + rawVectors.rewind(); + for (long rowPosition = 0; rowPosition < rawVectors.rowCount(); rowPosition++) { + boolean present = rawVectors.readNextVector(vector); + if (!present || source.excludedPosition.test(rowPosition)) { + continue; + } + singleColumnWriter.write(vector, fileOffset + rowPosition); + liveRowCount++; + } + } finally { + source.closeReader(rawVectors); + } + fileOffset += source.sourceFile.rowCount(); + } + + List results = writer.finish(); + checkArgument( + results.size() == 1, + "ANN segment build must produce exactly one payload file, but produced %s.", + results.size()); + ResultEntry result = results.get(0); + Path payloadPath = fileWriter.path(result.fileName()); + byte[] payloadMetadata = result.meta() == null ? new byte[0] : result.meta(); + PkVectorSegmentMeta metadata = + new PkVectorSegmentMeta( + ANN, + indexDefinitionId, + vectorField.id(), + vectorTypeFingerprint, + normalizeMetric(metric), + algorithm, + sourceFiles, + ordinalLayout, + liveRowCount, + buildSnapshotId, + optionsHash, + payloadMetadata); + IndexFileMeta segment = + new IndexFileMeta( + PK_VECTOR_ANN, + result.fileName(), + fileIO.getFileSize(payloadPath), + liveRowCount, + new GlobalIndexMeta( + 0, totalRowCount, vectorField.id(), null, metadata.serialize()), + pathFactory.isExternalPath() ? payloadPath.toString() : null); + success = true; + return segment; + } finally { + if (writer instanceof AutoCloseable) { + IOUtils.closeQuietly((AutoCloseable) writer); + } + if (!success) { + fileWriter.deleteCreatedFiles(); + } + } + } + + private static PkVectorSegmentMeta.SourceFile sourceMetadata(DataFileMeta sourceFile) { + return new PkVectorSegmentMeta.SourceFile( + sourceFile.fileName(), + sourceFile.schemaId(), + sourceFile.level(), + sourceFile.rowCount(), + sourceFile.fileSize()); + } + + private static String normalizeMetric(String metric) { + return metric.toLowerCase(Locale.ROOT).replace('-', '_'); + } + + private class SegmentFileWriter implements GlobalIndexFileWriter { + + private final Map createdFiles = new HashMap<>(); + + @Override + public String newFileName(String prefix) { + Path path = pathFactory.newPath(); + createdFiles.put(path.getName(), path); + return path.getName(); + } + + @Override + public PositionOutputStream newOutputStream(String fileName) throws IOException { + return fileIO.newOutputStream(path(fileName), false); + } + + private Path path(String fileName) { + Path path = createdFiles.get(fileName); + checkArgument(path != null, "ANN payload file %s was not allocated.", fileName); + return path; + } + + private void deleteCreatedFiles() { + for (Path path : createdFiles.values()) { + fileIO.deleteQuietly(path); + } + } + } + + /** One raw vector source used while building an ANN segment. */ + public static class Source { + + private final PkVectorSegmentMeta.SourceFile sourceFile; + @Nullable private final RawVectorSidecarReader rawVectors; + @Nullable private final ReaderFactory readerFactory; + private final LongPredicate excludedPosition; + + public Source(DataFileMeta sourceFile, RawVectorSidecarReader rawVectors) { + this(sourceFile, rawVectors, position -> false); + } + + public Source( + DataFileMeta sourceFile, + RawVectorSidecarReader rawVectors, + LongPredicate excludedPosition) { + this(sourceMetadata(sourceFile), rawVectors, excludedPosition); + } + + Source( + PkVectorSegmentMeta.SourceFile sourceFile, + RawVectorSidecarReader rawVectors, + LongPredicate excludedPosition) { + this.sourceFile = sourceFile; + this.rawVectors = rawVectors; + this.readerFactory = null; + this.excludedPosition = excludedPosition; + } + + private Source( + PkVectorSegmentMeta.SourceFile sourceFile, + ReaderFactory readerFactory, + LongPredicate excludedPosition) { + this.sourceFile = sourceFile; + this.rawVectors = null; + this.readerFactory = readerFactory; + this.excludedPosition = excludedPosition; + } + + static Source lazy(PkVectorSegmentMeta.SourceFile sourceFile, ReaderFactory readerFactory) { + return new Source(sourceFile, readerFactory, position -> false); + } + + private RawVectorSidecarReader openReader() throws IOException { + return rawVectors != null ? rawVectors : readerFactory.open(); + } + + private void closeReader(RawVectorSidecarReader reader) throws IOException { + if (rawVectors == null) { + reader.close(); + } + } + + @FunctionalInterface + interface ReaderFactory { + RawVectorSidecarReader open() throws IOException; + } + } +} diff --git a/paimon-core/src/main/java/org/apache/paimon/index/pkvector/PkVectorAnnSegmentSearcher.java b/paimon-core/src/main/java/org/apache/paimon/index/pkvector/PkVectorAnnSegmentSearcher.java new file mode 100644 index 000000000000..2ba7daae3596 --- /dev/null +++ b/paimon-core/src/main/java/org/apache/paimon/index/pkvector/PkVectorAnnSegmentSearcher.java @@ -0,0 +1,291 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.index.pkvector; + +import org.apache.paimon.deletionvectors.DeletionVector; +import org.apache.paimon.fs.FileIO; +import org.apache.paimon.globalindex.GlobalIndexIOMeta; +import org.apache.paimon.globalindex.GlobalIndexReader; +import org.apache.paimon.globalindex.GlobalIndexer; +import org.apache.paimon.globalindex.ScoredGlobalIndexResult; +import org.apache.paimon.globalindex.VectorGlobalIndexer; +import org.apache.paimon.index.IndexFileMeta; +import org.apache.paimon.options.Options; +import org.apache.paimon.predicate.VectorSearch; +import org.apache.paimon.types.DataField; +import org.apache.paimon.utils.IOUtils; +import org.apache.paimon.utils.Range; +import org.apache.paimon.utils.RoaringNavigableMap64; + +import javax.annotation.Nullable; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.ExecutorService; + +import static org.apache.paimon.index.pkvector.PkVectorSegmentMeta.OrdinalLayout.FILE_POSITION; +import static org.apache.paimon.index.pkvector.PkVectorSegmentMeta.OrdinalLayout.ROW_POSITION; +import static org.apache.paimon.index.pkvector.PkVectorSegmentMeta.Role.ANN; +import static org.apache.paimon.utils.Preconditions.checkArgument; + +/** Searches one ANN payload and maps its segment-local ids back to source row positions. */ +public class PkVectorAnnSegmentSearcher { + + private static final Comparator BEST_FIRST = + Comparator.comparingDouble((Candidate candidate) -> candidate.distance) + .thenComparing(candidate -> candidate.dataFileName) + .thenComparingLong(candidate -> candidate.rowPosition); + + private final FileIO fileIO; + private final PkVectorAnnSegmentFile annSegmentFile; + private final DataField vectorField; + private final Options indexOptions; + private final ExecutorService executor; + + public PkVectorAnnSegmentSearcher( + FileIO fileIO, + PkVectorAnnSegmentFile annSegmentFile, + DataField vectorField, + Options indexOptions, + ExecutorService executor) { + this.fileIO = fileIO; + this.annSegmentFile = annSegmentFile; + this.vectorField = vectorField; + this.indexOptions = indexOptions; + this.executor = executor; + } + + public List search( + IndexFileMeta segment, + PkVectorSegmentMeta metadata, + float[] query, + int limit, + @Nullable DeletionVector deletionVector, + Map searchOptions) { + Map deletionVectors = new HashMap<>(); + if (deletionVector != null) { + checkArgument( + metadata.sourceFiles().size() == 1, + "A single deletion vector can only search a single-source ANN segment."); + deletionVectors.put(metadata.sourceFiles().get(0).fileName(), deletionVector); + } + return search(segment, metadata, query, limit, deletionVectors, searchOptions); + } + + public List search( + IndexFileMeta segment, + PkVectorSegmentMeta metadata, + float[] query, + int limit, + Map deletionVectors, + Map searchOptions) { + checkArgument(limit > 0, "Vector search limit must be positive: %s.", limit); + checkArgument( + PkVectorAnnSegmentFile.PK_VECTOR_ANN.equals(segment.indexType()), + "Vector segment %s is not an ANN payload.", + segment.fileName()); + checkArgument(metadata.role() == ANN, "Vector segment %s is not ANN.", segment.fileName()); + checkArgument( + metadata.ordinalLayout() == ROW_POSITION + || metadata.ordinalLayout() == FILE_POSITION, + "ANN segment %s has unsupported ordinal layout %s.", + segment.fileName(), + metadata.ordinalLayout()); + checkArgument( + metadata.ordinalLayout() != ROW_POSITION || metadata.sourceFiles().size() == 1, + "Row-position ANN segment %s must reference exactly one source file.", + segment.fileName()); + checkArgument( + metadata.vectorFieldId() == vectorField.id(), + "ANN segment %s has vector field %s, but reader expects %s.", + segment.fileName(), + metadata.vectorFieldId(), + vectorField.id()); + + GlobalIndexer indexer = + GlobalIndexer.create(metadata.algorithm(), vectorField, indexOptions); + checkArgument( + indexer instanceof VectorGlobalIndexer, + "Index algorithm %s does not implement VectorGlobalIndexer.", + metadata.algorithm()); + String metric = normalizeMetric(metadata.metric()); + String readerMetric = normalizeMetric(((VectorGlobalIndexer) indexer).metric()); + checkArgument( + metric.equals(readerMetric), + "ANN segment metric %s does not match index reader metric %s.", + metric, + readerMetric); + + GlobalIndexIOMeta ioMeta = + new GlobalIndexIOMeta( + annSegmentFile.path(segment), + segment.fileSize(), + metadata.payloadMetadata()); + GlobalIndexReader reader = + indexer.createReader( + meta -> fileIO.newInputStream(meta.filePath()), + Collections.singletonList(ioMeta), + executor); + try { + VectorSearch search = new VectorSearch(query, limit, vectorField.name(), searchOptions); + RoaringNavigableMap64 liveRows = + liveRowPositions(metadata.sourceFiles(), deletionVectors); + if (liveRows != null) { + search.withIncludeRowIds(liveRows); + } + Optional result = reader.visitVectorSearch(search).join(); + if (!result.isPresent()) { + return Collections.emptyList(); + } + + long sourceRowCount = totalRowCount(metadata.sourceFiles()); + List candidates = new ArrayList<>(); + ScoredGlobalIndexResult scored = result.get(); + for (long ordinal : scored.results()) { + checkArgument( + ordinal >= 0 && ordinal < sourceRowCount, + "ANN segment %s returned ordinal %s outside [0, %s).", + segment.fileName(), + ordinal, + sourceRowCount); + FilePosition filePosition = filePosition(metadata.sourceFiles(), ordinal); + DeletionVector deletionVector = deletionVectors.get(filePosition.dataFileName); + checkArgument( + deletionVector == null + || !deletionVector.isDeleted(filePosition.rowPosition), + "ANN segment %s returned snapshot-deleted row position %s.", + segment.fileName(), + filePosition.rowPosition); + candidates.add( + new Candidate( + filePosition.dataFileName, + filePosition.rowPosition, + scoreToDistance(scored.scoreGetter().score(ordinal), metric))); + } + Collections.sort(candidates, BEST_FIRST); + return Collections.unmodifiableList(candidates); + } finally { + IOUtils.closeQuietly(reader); + } + } + + @Nullable + private static RoaringNavigableMap64 liveRowPositions( + List sourceFiles, + Map deletionVectors) { + if (deletionVectors.isEmpty()) { + return null; + } + RoaringNavigableMap64 live = new RoaringNavigableMap64(); + RoaringNavigableMap64 deleted = new RoaringNavigableMap64(); + long fileOffset = 0; + for (PkVectorSegmentMeta.SourceFile sourceFile : sourceFiles) { + if (sourceFile.rowCount() > 0) { + live.addRange(new Range(fileOffset, fileOffset + sourceFile.rowCount() - 1)); + } + DeletionVector deletionVector = deletionVectors.get(sourceFile.fileName()); + if (deletionVector != null) { + final long offset = fileOffset; + deletionVector.forEachDeletedPosition(position -> deleted.add(offset + position)); + } + fileOffset += sourceFile.rowCount(); + } + live.andNot(deleted); + return live; + } + + private static long totalRowCount(List sourceFiles) { + long total = 0; + for (PkVectorSegmentMeta.SourceFile sourceFile : sourceFiles) { + total = Math.addExact(total, sourceFile.rowCount()); + } + return total; + } + + private static FilePosition filePosition( + List sourceFiles, long ordinal) { + long fileOffset = 0; + for (PkVectorSegmentMeta.SourceFile sourceFile : sourceFiles) { + long nextOffset = fileOffset + sourceFile.rowCount(); + if (ordinal < nextOffset) { + return new FilePosition(sourceFile.fileName(), ordinal - fileOffset); + } + fileOffset = nextOffset; + } + throw new IllegalArgumentException("ANN ordinal is outside source files: " + ordinal); + } + + private static float scoreToDistance(float score, String metric) { + if ("l2".equals(metric)) { + return 1F / score - 1F; + } else if ("cosine".equals(metric)) { + return 1F - score; + } else if ("inner_product".equals(metric)) { + return -score; + } + throw new IllegalArgumentException("Unsupported ANN vector metric: " + metric); + } + + private static String normalizeMetric(String metric) { + return metric.toLowerCase(Locale.ROOT).replace('-', '_'); + } + + /** One ANN candidate addressed by source-file row position. */ + public static class Candidate { + + private final long rowPosition; + private final float distance; + private final String dataFileName; + + private Candidate(String dataFileName, long rowPosition, float distance) { + this.dataFileName = dataFileName; + this.rowPosition = rowPosition; + this.distance = distance; + } + + public String dataFileName() { + return dataFileName; + } + + public long rowPosition() { + return rowPosition; + } + + public float distance() { + return distance; + } + } + + private static class FilePosition { + + private final String dataFileName; + private final long rowPosition; + + private FilePosition(String dataFileName, long rowPosition) { + this.dataFileName = dataFileName; + this.rowPosition = rowPosition; + } + } +} diff --git a/paimon-core/src/main/java/org/apache/paimon/index/pkvector/PkVectorRawSegmentFile.java b/paimon-core/src/main/java/org/apache/paimon/index/pkvector/PkVectorRawSegmentFile.java new file mode 100644 index 000000000000..657363a4decd --- /dev/null +++ b/paimon-core/src/main/java/org/apache/paimon/index/pkvector/PkVectorRawSegmentFile.java @@ -0,0 +1,208 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.index.pkvector; + +import org.apache.paimon.data.InternalVector; +import org.apache.paimon.fs.FileIO; +import org.apache.paimon.fs.Path; +import org.apache.paimon.index.GlobalIndexMeta; +import org.apache.paimon.index.IndexFile; +import org.apache.paimon.index.IndexFileMeta; +import org.apache.paimon.index.IndexPathFactory; +import org.apache.paimon.io.DataFileMeta; +import org.apache.paimon.io.FileWriterAbortExecutor; +import org.apache.paimon.io.KeyValueVectorSidecarWriter; +import org.apache.paimon.manifest.FileSource; + +import javax.annotation.Nullable; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.util.Collections; +import java.util.Optional; +import java.util.function.BiConsumer; +import java.util.function.Consumer; + +import static org.apache.paimon.index.pkvector.PkVectorSegmentMeta.OrdinalLayout.ROW_POSITION; +import static org.apache.paimon.index.pkvector.PkVectorSegmentMeta.Role.RAW_DELTA; +import static org.apache.paimon.utils.Preconditions.checkArgument; +import static org.apache.paimon.utils.Preconditions.checkState; + +/** Creates immutable row-position-addressable raw vector segments for data files. */ +public class PkVectorRawSegmentFile extends IndexFile { + + public static final String PK_VECTOR_RAW = "pk-vector-raw"; + + public PkVectorRawSegmentFile(FileIO fileIO, IndexPathFactory pathFactory) { + super(fileIO, pathFactory); + } + + RawVectorSidecarReader newReader(IndexFileMeta segment) throws IOException { + return new RawVectorSidecarReader(fileIO, path(segment)); + } + + public KeyValueVectorSidecarWriter newWriter( + int dimension, + String indexDefinitionId, + int vectorFieldId, + String vectorTypeFingerprint, + String metric, + String algorithm, + byte[] optionsHash, + BiConsumer segmentConsumer, + Consumer segmentAbortConsumer) { + Path path = pathFactory.newPath(); + try { + return new Writer( + new RawVectorSidecarWriter(fileIO, path, dimension), + indexDefinitionId, + vectorFieldId, + vectorTypeFingerprint, + metric, + algorithm, + optionsHash, + segmentConsumer, + segmentAbortConsumer); + } catch (IOException e) { + fileIO.deleteQuietly(path); + throw new UncheckedIOException( + "Failed to create primary-key raw vector segment: " + path, e); + } + } + + private class Writer implements KeyValueVectorSidecarWriter { + + private final RawVectorSidecarWriter rawWriter; + private final String indexDefinitionId; + private final int vectorFieldId; + private final String vectorTypeFingerprint; + private final String metric; + private final String algorithm; + private final byte[] optionsHash; + private final BiConsumer segmentConsumer; + private final Consumer segmentAbortConsumer; + + @Nullable private IndexFileMeta completedSegment; + private boolean closed; + private boolean completed; + + private Writer( + RawVectorSidecarWriter rawWriter, + String indexDefinitionId, + int vectorFieldId, + String vectorTypeFingerprint, + String metric, + String algorithm, + byte[] optionsHash, + BiConsumer segmentConsumer, + Consumer segmentAbortConsumer) { + this.rawWriter = rawWriter; + this.indexDefinitionId = indexDefinitionId; + this.vectorFieldId = vectorFieldId; + this.vectorTypeFingerprint = vectorTypeFingerprint; + this.metric = metric; + this.algorithm = algorithm; + this.optionsHash = optionsHash.clone(); + this.segmentConsumer = segmentConsumer; + this.segmentAbortConsumer = segmentAbortConsumer; + } + + @Override + public void write(@Nullable InternalVector vector) throws IOException { + rawWriter.write(vector); + } + + @Override + public void close() throws IOException { + if (!closed) { + rawWriter.close(); + closed = true; + } + } + + @Override + public void complete(DataFileMeta sourceFile) throws IOException { + checkState(closed, "Raw vector segment must be closed before completion."); + checkState(!completed, "Raw vector segment is already completed."); + checkArgument( + rawWriter.rowCount() == sourceFile.rowCount(), + "Raw vector segment row count %s does not match source file %s row count %s.", + rawWriter.rowCount(), + sourceFile.fileName(), + sourceFile.rowCount()); + + PkVectorSegmentMeta metadata = + new PkVectorSegmentMeta( + RAW_DELTA, + indexDefinitionId, + vectorFieldId, + vectorTypeFingerprint, + metric, + algorithm, + Collections.singletonList( + new PkVectorSegmentMeta.SourceFile( + sourceFile.fileName(), + sourceFile.schemaId(), + sourceFile.level(), + sourceFile.rowCount(), + sourceFile.fileSize())), + ROW_POSITION, + rawWriter.liveVectorCount(), + 0, + optionsHash); + Path path = rawWriter.path(); + IndexFileMeta segment = + new IndexFileMeta( + PK_VECTOR_RAW, + path.getName(), + fileIO.getFileSize(path), + rawWriter.liveVectorCount(), + new GlobalIndexMeta( + 0, + sourceFile.rowCount(), + vectorFieldId, + null, + metadata.serialize()), + pathFactory.isExternalPath() ? path.toString() : null); + segmentConsumer.accept(segment, sourceFile.fileSource().orElse(FileSource.APPEND)); + completedSegment = segment; + completed = true; + } + + @Override + public void abort() { + if (completedSegment != null) { + segmentAbortConsumer.accept(completedSegment); + completedSegment = null; + } + rawWriter.abort(); + } + + @Override + public Optional abortExecutor() { + return Optional.of( + new FileWriterAbortExecutor(fileIO, rawWriter.path()) { + @Override + public void abort() { + Writer.this.abort(); + } + }); + } + } +} diff --git a/paimon-core/src/main/java/org/apache/paimon/index/pkvector/PkVectorSegmentMeta.java b/paimon-core/src/main/java/org/apache/paimon/index/pkvector/PkVectorSegmentMeta.java new file mode 100644 index 000000000000..c9831f741537 --- /dev/null +++ b/paimon-core/src/main/java/org/apache/paimon/index/pkvector/PkVectorSegmentMeta.java @@ -0,0 +1,334 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.index.pkvector; + +import org.apache.paimon.io.DataInputDeserializer; +import org.apache.paimon.io.DataOutputSerializer; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Objects; + +import static org.apache.paimon.utils.Preconditions.checkArgument; + +/** Versioned metadata for an immutable primary-key vector segment. */ +public class PkVectorSegmentMeta { + + private static final int VERSION = 1; + + private final Role role; + private final String indexDefinitionId; + private final int vectorFieldId; + private final String vectorTypeFingerprint; + private final String metric; + private final String algorithm; + private final List sourceFiles; + private final OrdinalLayout ordinalLayout; + private final long liveRowCountAtBuild; + private final long buildSnapshotId; + private final byte[] optionsHash; + private final byte[] payloadMetadata; + + public PkVectorSegmentMeta( + Role role, + String indexDefinitionId, + int vectorFieldId, + String vectorTypeFingerprint, + String metric, + String algorithm, + List sourceFiles, + OrdinalLayout ordinalLayout, + long liveRowCountAtBuild, + long buildSnapshotId, + byte[] optionsHash) { + this( + role, + indexDefinitionId, + vectorFieldId, + vectorTypeFingerprint, + metric, + algorithm, + sourceFiles, + ordinalLayout, + liveRowCountAtBuild, + buildSnapshotId, + optionsHash, + new byte[0]); + } + + public PkVectorSegmentMeta( + Role role, + String indexDefinitionId, + int vectorFieldId, + String vectorTypeFingerprint, + String metric, + String algorithm, + List sourceFiles, + OrdinalLayout ordinalLayout, + long liveRowCountAtBuild, + long buildSnapshotId, + byte[] optionsHash, + byte[] payloadMetadata) { + this.role = Objects.requireNonNull(role); + this.indexDefinitionId = Objects.requireNonNull(indexDefinitionId); + this.vectorFieldId = vectorFieldId; + this.vectorTypeFingerprint = Objects.requireNonNull(vectorTypeFingerprint); + this.metric = Objects.requireNonNull(metric); + this.algorithm = Objects.requireNonNull(algorithm); + this.sourceFiles = Collections.unmodifiableList(new ArrayList<>(sourceFiles)); + this.ordinalLayout = Objects.requireNonNull(ordinalLayout); + this.liveRowCountAtBuild = liveRowCountAtBuild; + this.buildSnapshotId = buildSnapshotId; + this.optionsHash = Arrays.copyOf(optionsHash, optionsHash.length); + this.payloadMetadata = Arrays.copyOf(payloadMetadata, payloadMetadata.length); + + checkArgument(!this.sourceFiles.isEmpty(), "A vector segment must reference source files."); + checkArgument(liveRowCountAtBuild >= 0, "Live row count must not be negative."); + checkArgument(buildSnapshotId >= 0, "Build snapshot id must not be negative."); + } + + public Role role() { + return role; + } + + public String indexDefinitionId() { + return indexDefinitionId; + } + + public int vectorFieldId() { + return vectorFieldId; + } + + public String vectorTypeFingerprint() { + return vectorTypeFingerprint; + } + + public String metric() { + return metric; + } + + public String algorithm() { + return algorithm; + } + + public List sourceFiles() { + return sourceFiles; + } + + public OrdinalLayout ordinalLayout() { + return ordinalLayout; + } + + public long liveRowCountAtBuild() { + return liveRowCountAtBuild; + } + + public long buildSnapshotId() { + return buildSnapshotId; + } + + public byte[] optionsHash() { + return Arrays.copyOf(optionsHash, optionsHash.length); + } + + public byte[] payloadMetadata() { + return Arrays.copyOf(payloadMetadata, payloadMetadata.length); + } + + /** Serializes this metadata for {@link org.apache.paimon.index.GlobalIndexMeta#indexMeta()}. */ + public byte[] serialize() { + try { + DataOutputSerializer output = new DataOutputSerializer(128); + output.writeInt(VERSION); + output.writeByte(role.ordinal()); + output.writeUTF(indexDefinitionId); + output.writeInt(vectorFieldId); + output.writeUTF(vectorTypeFingerprint); + output.writeUTF(metric); + output.writeUTF(algorithm); + output.writeInt(sourceFiles.size()); + for (SourceFile sourceFile : sourceFiles) { + output.writeUTF(sourceFile.fileName); + output.writeLong(sourceFile.schemaId); + output.writeInt(sourceFile.level); + output.writeLong(sourceFile.rowCount); + output.writeLong(sourceFile.fileSize); + } + output.writeByte(ordinalLayout.ordinal()); + output.writeLong(liveRowCountAtBuild); + output.writeLong(buildSnapshotId); + output.writeInt(optionsHash.length); + output.write(optionsHash); + output.writeInt(payloadMetadata.length); + output.write(payloadMetadata); + return output.getCopyOfBuffer(); + } catch (IOException e) { + throw new RuntimeException( + "Failed to serialize primary-key vector segment metadata.", e); + } + } + + /** Deserializes primary-key vector metadata stored in {@code GlobalIndexMeta.indexMeta}. */ + public static PkVectorSegmentMeta deserialize(byte[] bytes) { + try { + DataInputDeserializer input = new DataInputDeserializer(bytes); + int version = input.readInt(); + checkArgument( + version == VERSION, + "Unsupported primary-key vector segment version: %s.", + version); + Role role = enumValue(Role.values(), input.readByte(), "role"); + String indexDefinitionId = input.readUTF(); + int vectorFieldId = input.readInt(); + String vectorTypeFingerprint = input.readUTF(); + String metric = input.readUTF(); + String algorithm = input.readUTF(); + int sourceFileCount = input.readInt(); + checkArgument(sourceFileCount > 0, "A vector segment must reference source files."); + List sourceFiles = new ArrayList<>(sourceFileCount); + for (int i = 0; i < sourceFileCount; i++) { + sourceFiles.add( + new SourceFile( + input.readUTF(), + input.readLong(), + input.readInt(), + input.readLong(), + input.readLong())); + } + OrdinalLayout ordinalLayout = + enumValue(OrdinalLayout.values(), input.readByte(), "ordinal layout"); + long liveRowCountAtBuild = input.readLong(); + long buildSnapshotId = input.readLong(); + int optionsHashLength = input.readInt(); + checkArgument(optionsHashLength >= 0, "Options hash length must not be negative."); + byte[] optionsHash = new byte[optionsHashLength]; + input.readFully(optionsHash); + int payloadMetadataLength = input.readInt(); + checkArgument( + payloadMetadataLength >= 0, "Payload metadata length must not be negative."); + byte[] payloadMetadata = new byte[payloadMetadataLength]; + input.readFully(payloadMetadata); + checkArgument( + input.available() == 0, + "Unexpected trailing bytes in vector segment metadata."); + return new PkVectorSegmentMeta( + role, + indexDefinitionId, + vectorFieldId, + vectorTypeFingerprint, + metric, + algorithm, + sourceFiles, + ordinalLayout, + liveRowCountAtBuild, + buildSnapshotId, + optionsHash, + payloadMetadata); + } catch (IOException e) { + throw new IllegalArgumentException( + "Failed to deserialize primary-key vector segment metadata.", e); + } + } + + private static T enumValue(T[] values, byte ordinal, String field) { + int index = ordinal; + checkArgument( + index >= 0 && index < values.length, + "Unknown vector segment %s: %s.", + field, + ordinal); + return values[index]; + } + + /** Role of an immutable vector payload. */ + public enum Role { + RAW_DELTA, + ANN + } + + /** Mapping from a segment-local ordinal to a physical data-file position. */ + public enum OrdinalLayout { + ROW_POSITION, + FILE_POSITION + } + + /** Immutable source data-file identity captured when a vector segment is built. */ + public static class SourceFile { + + private final String fileName; + private final long schemaId; + private final int level; + private final long rowCount; + private final long fileSize; + + public SourceFile(String fileName, long schemaId, int level, long rowCount, long fileSize) { + this.fileName = Objects.requireNonNull(fileName); + this.schemaId = schemaId; + this.level = level; + this.rowCount = rowCount; + this.fileSize = fileSize; + checkArgument(rowCount >= 0, "Source file row count must not be negative."); + checkArgument(fileSize >= 0, "Source file size must not be negative."); + } + + public String fileName() { + return fileName; + } + + public long schemaId() { + return schemaId; + } + + public int level() { + return level; + } + + public long rowCount() { + return rowCount; + } + + public long fileSize() { + return fileSize; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + SourceFile that = (SourceFile) o; + return schemaId == that.schemaId + && level == that.level + && rowCount == that.rowCount + && fileSize == that.fileSize + && Objects.equals(fileName, that.fileName); + } + + @Override + public int hashCode() { + return Objects.hash(fileName, schemaId, level, rowCount, fileSize); + } + } +} diff --git a/paimon-core/src/main/java/org/apache/paimon/index/pkvector/RawVectorSidecarReader.java b/paimon-core/src/main/java/org/apache/paimon/index/pkvector/RawVectorSidecarReader.java new file mode 100644 index 000000000000..7d77c05fe816 --- /dev/null +++ b/paimon-core/src/main/java/org/apache/paimon/index/pkvector/RawVectorSidecarReader.java @@ -0,0 +1,247 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.index.pkvector; + +import org.apache.paimon.fs.FileIO; +import org.apache.paimon.fs.Path; +import org.apache.paimon.fs.SeekableInputStream; + +import java.io.Closeable; +import java.io.DataInputStream; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.List; +import java.util.PriorityQueue; +import java.util.function.LongPredicate; + +import static org.apache.paimon.index.pkvector.RawVectorSidecarWriter.HEADER_SIZE; +import static org.apache.paimon.index.pkvector.RawVectorSidecarWriter.MAGIC; +import static org.apache.paimon.index.pkvector.RawVectorSidecarWriter.VERSION; +import static org.apache.paimon.index.pkvector.RawVectorSidecarWriter.recordSize; +import static org.apache.paimon.utils.Preconditions.checkArgument; + +/** Random-access reader for raw vector sidecars. This class is not thread-safe. */ +public class RawVectorSidecarReader implements Closeable { + + private static final Comparator BEST_FIRST = + (left, right) -> { + int distance = Float.compare(left.distance, right.distance); + return distance != 0 ? distance : Long.compare(left.rowPosition, right.rowPosition); + }; + + private final SeekableInputStream input; + private final DataInputStream dataInput; + private final int dimension; + private final int recordSize; + private final long rowCount; + private long nextSequentialPosition; + + public RawVectorSidecarReader(FileIO fileIO, Path path) throws IOException { + long fileSize = fileIO.getFileSize(path); + checkArgument( + fileSize >= HEADER_SIZE, "Raw vector sidecar %s is shorter than its header.", path); + this.input = fileIO.newInputStream(path); + this.dataInput = new DataInputStream(input); + int magic = dataInput.readInt(); + checkArgument(magic == MAGIC, "File %s is not a raw vector sidecar.", path); + int version = dataInput.readInt(); + checkArgument(version == VERSION, "Unsupported raw vector sidecar version: %s.", version); + this.dimension = dataInput.readInt(); + this.recordSize = dataInput.readInt(); + checkArgument(dimension > 0, "Raw vector sidecar dimension must be positive."); + checkArgument( + recordSize == recordSize(dimension), + "Raw vector sidecar record size %s does not match dimension %s.", + recordSize, + dimension); + long payloadSize = fileSize - HEADER_SIZE; + checkArgument( + payloadSize % recordSize == 0, + "Raw vector sidecar %s has a truncated record.", + path); + this.rowCount = payloadSize / recordSize; + this.nextSequentialPosition = 0; + } + + public int dimension() { + return dimension; + } + + public long rowCount() { + return rowCount; + } + + /** Raw sidecars use row positions directly as segment ordinals. */ + public long rowPositionForOrdinal(long ordinal) { + checkRowPosition(ordinal); + return ordinal; + } + + public float[] readVector(long rowPosition) throws IOException { + checkRowPosition(rowPosition); + nextSequentialPosition = -1; + input.seek(HEADER_SIZE + rowPosition * recordSize); + if (!dataInput.readBoolean()) { + return null; + } + float[] vector = new float[dimension]; + for (int i = 0; i < dimension; i++) { + vector[i] = dataInput.readFloat(); + } + return vector; + } + + /** Positions this reader for a sequential pass over all row-position records. */ + public void rewind() throws IOException { + input.seek(HEADER_SIZE); + nextSequentialPosition = 0; + } + + /** + * Reads the next record into a caller-owned reusable buffer and returns whether it is non-null. + */ + public boolean readNextVector(float[] reuse) throws IOException { + checkArgument( + reuse.length == dimension, + "Reusable vector buffer dimension must be %s, but is %s.", + dimension, + reuse.length); + checkArgument( + nextSequentialPosition >= 0, + "Raw vector sequential read requires rewind after random access."); + checkArgument( + nextSequentialPosition < rowCount, + "No raw vector remains after row position %s.", + nextSequentialPosition); + boolean present = dataInput.readBoolean(); + for (int i = 0; i < dimension; i++) { + reuse[i] = dataInput.readFloat(); + } + nextSequentialPosition++; + return present; + } + + /** Performs an exact top-k scan and excludes positions deleted in the selected snapshot. */ + public List search( + float[] query, String metric, int limit, LongPredicate excludedPosition) + throws IOException { + checkArgument(query.length == dimension, "Query vector dimension must be %s.", dimension); + checkArgument(limit > 0, "Vector search limit must be positive: %s.", limit); + checkArgument( + "l2".equals(metric) || "cosine".equals(metric) || "inner_product".equals(metric), + "Unsupported raw vector distance metric: %s.", + metric); + for (int i = 0; i < query.length; i++) { + checkArgument( + !Float.isNaN(query[i]) && !Float.isInfinite(query[i]), + "Query vector element at index %s must be finite.", + i); + } + + PriorityQueue nearest = new PriorityQueue<>(limit, BEST_FIRST.reversed()); + float[] vector = new float[dimension]; + nextSequentialPosition = -1; + input.seek(HEADER_SIZE); + for (long rowPosition = 0; rowPosition < rowCount; rowPosition++) { + boolean present = dataInput.readBoolean(); + for (int i = 0; i < dimension; i++) { + vector[i] = dataInput.readFloat(); + } + if (!present || excludedPosition.test(rowPosition)) { + continue; + } + + Candidate candidate = new Candidate(rowPosition, distance(query, vector, metric)); + if (nearest.size() < limit) { + nearest.add(candidate); + } else if (BEST_FIRST.compare(candidate, nearest.peek()) < 0) { + nearest.poll(); + nearest.add(candidate); + } + } + + List result = new ArrayList<>(nearest); + Collections.sort(result, BEST_FIRST); + return result; + } + + @Override + public void close() throws IOException { + dataInput.close(); + } + + private void checkRowPosition(long rowPosition) { + checkArgument( + rowPosition >= 0 && rowPosition < rowCount, + "Raw vector row position %s is outside [0, %s).", + rowPosition, + rowCount); + } + + private float distance(float[] query, float[] vector, String metric) { + if ("l2".equals(metric)) { + double squaredDistance = 0; + for (int i = 0; i < dimension; i++) { + double delta = vector[i] - query[i]; + squaredDistance += delta * delta; + } + return (float) squaredDistance; + } + + double dot = 0; + double queryNorm = 0; + double vectorNorm = 0; + for (int i = 0; i < dimension; i++) { + dot += query[i] * vector[i]; + queryNorm += query[i] * query[i]; + vectorNorm += vector[i] * vector[i]; + } + if ("inner_product".equals(metric)) { + return (float) -dot; + } + if (queryNorm == 0 || vectorNorm == 0) { + return 1; + } + double similarity = dot / Math.sqrt(queryNorm * vectorNorm); + similarity = Math.max(-1, Math.min(1, similarity)); + return (float) (1 - similarity); + } + + /** One exact raw-vector candidate. Lower distance is better. */ + public static class Candidate { + + private final long rowPosition; + private final float distance; + + private Candidate(long rowPosition, float distance) { + this.rowPosition = rowPosition; + this.distance = distance; + } + + public long rowPosition() { + return rowPosition; + } + + public float distance() { + return distance; + } + } +} diff --git a/paimon-core/src/main/java/org/apache/paimon/index/pkvector/RawVectorSidecarWriter.java b/paimon-core/src/main/java/org/apache/paimon/index/pkvector/RawVectorSidecarWriter.java new file mode 100644 index 000000000000..27ad85e2670e --- /dev/null +++ b/paimon-core/src/main/java/org/apache/paimon/index/pkvector/RawVectorSidecarWriter.java @@ -0,0 +1,163 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.index.pkvector; + +import org.apache.paimon.data.InternalArray; +import org.apache.paimon.fs.FileIO; +import org.apache.paimon.fs.Path; +import org.apache.paimon.fs.PositionOutputStream; + +import javax.annotation.Nullable; + +import java.io.Closeable; +import java.io.DataOutputStream; +import java.io.IOException; + +import static org.apache.paimon.utils.Preconditions.checkArgument; +import static org.apache.paimon.utils.Preconditions.checkState; + +/** Streaming writer for a row-position-addressable raw float-vector sidecar. */ +public class RawVectorSidecarWriter implements Closeable { + + static final int MAGIC = 0x50565231; + static final int VERSION = 1; + static final int HEADER_SIZE = 16; + + private final FileIO fileIO; + private final Path path; + private final int dimension; + private final DataOutputStream output; + private final float[] vectorBuffer; + + private long rowCount; + private long liveVectorCount; + private boolean closed; + + public RawVectorSidecarWriter(FileIO fileIO, Path path, int dimension) throws IOException { + checkArgument(dimension > 0, "Raw vector dimension must be positive: %s.", dimension); + checkArgument( + dimension <= (Integer.MAX_VALUE - 1) / Float.BYTES, + "Raw vector dimension is too large: %s.", + dimension); + this.fileIO = fileIO; + this.path = path; + this.dimension = dimension; + this.vectorBuffer = new float[dimension]; + PositionOutputStream stream = fileIO.newOutputStream(path, false); + this.output = new DataOutputStream(stream); + this.output.writeInt(MAGIC); + this.output.writeInt(VERSION); + this.output.writeInt(dimension); + this.output.writeInt(recordSize(dimension)); + } + + public void write(@Nullable Object vector) throws IOException { + checkState(!closed, "Raw vector sidecar writer is already closed."); + if (vector == null) { + output.writeBoolean(false); + for (int i = 0; i < dimension; i++) { + output.writeFloat(0); + } + rowCount++; + return; + } + + float[] values = materializeAndValidate(vector); + output.writeBoolean(true); + for (float value : values) { + output.writeFloat(value); + } + rowCount++; + liveVectorCount++; + } + + public long rowCount() { + return rowCount; + } + + public long liveVectorCount() { + return liveVectorCount; + } + + public Path path() { + return path; + } + + @Override + public void close() throws IOException { + if (!closed) { + closed = true; + output.close(); + } + } + + /** Best-effort cleanup used when the owning data-file writer fails or is aborted. */ + public void abort() { + try { + close(); + } catch (IOException ignored) { + // Keep abort best-effort and always try to remove the incomplete sidecar. + } + fileIO.deleteQuietly(path); + } + + static int recordSize(int dimension) { + return 1 + dimension * Float.BYTES; + } + + private void checkDimension(int actualDimension) { + checkArgument( + actualDimension == dimension, + "Raw vector dimension must be %s, but was %s.", + dimension, + actualDimension); + } + + private float[] materializeAndValidate(Object vector) { + if (vector instanceof float[]) { + float[] values = (float[]) vector; + checkDimension(values.length); + for (int i = 0; i < dimension; i++) { + checkFinite(values[i], i); + } + return values; + } + if (vector instanceof InternalArray) { + InternalArray values = (InternalArray) vector; + checkDimension(values.size()); + for (int i = 0; i < dimension; i++) { + checkArgument(!values.isNullAt(i), "Vector element at index %s is null.", i); + float value = values.getFloat(i); + checkFinite(value, i); + vectorBuffer[i] = value; + } + return vectorBuffer; + } + throw new IllegalArgumentException( + "Unsupported raw vector value type: " + vector.getClass().getName()); + } + + private static void checkFinite(float value, int index) { + checkArgument( + !Float.isNaN(value) && !Float.isInfinite(value), + "Vector element at index %s must be finite, but was %s.", + index, + value); + } +} diff --git a/paimon-core/src/main/java/org/apache/paimon/io/KeyValueVectorSidecarWriter.java b/paimon-core/src/main/java/org/apache/paimon/io/KeyValueVectorSidecarWriter.java new file mode 100644 index 000000000000..b61adb4666d4 --- /dev/null +++ b/paimon-core/src/main/java/org/apache/paimon/io/KeyValueVectorSidecarWriter.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.io; + +import org.apache.paimon.data.InternalVector; + +import javax.annotation.Nullable; + +import java.io.Closeable; +import java.io.IOException; +import java.util.Optional; + +/** Synchronous vector sidecar owned by one key-value data-file writer. */ +public interface KeyValueVectorSidecarWriter extends Closeable { + + void write(@Nullable InternalVector vector) throws IOException; + + void complete(DataFileMeta sourceFile) throws IOException; + + void abort(); + + /** Lightweight cleanup retained by a rolling writer after this sidecar is completed. */ + default Optional abortExecutor() { + return Optional.empty(); + } +} diff --git a/paimon-core/src/test/java/org/apache/paimon/index/pkvector/PkVectorAnnSegmentFileTest.java b/paimon-core/src/test/java/org/apache/paimon/index/pkvector/PkVectorAnnSegmentFileTest.java new file mode 100644 index 000000000000..fe1546467ca9 --- /dev/null +++ b/paimon-core/src/test/java/org/apache/paimon/index/pkvector/PkVectorAnnSegmentFileTest.java @@ -0,0 +1,320 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.index.pkvector; + +import org.apache.paimon.deletionvectors.BitmapDeletionVector; +import org.apache.paimon.fs.Path; +import org.apache.paimon.fs.local.LocalFileIO; +import org.apache.paimon.index.IndexFileMeta; +import org.apache.paimon.index.IndexPathFactory; +import org.apache.paimon.io.DataFileMeta; +import org.apache.paimon.manifest.FileSource; +import org.apache.paimon.options.Options; +import org.apache.paimon.stats.SimpleStats; +import org.apache.paimon.types.DataField; +import org.apache.paimon.types.DataTypes; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; + +import static org.apache.paimon.index.pkvector.PkVectorSegmentMeta.OrdinalLayout.ROW_POSITION; +import static org.apache.paimon.index.pkvector.PkVectorSegmentMeta.Role.ANN; +import static org.assertj.core.api.Assertions.assertThat; + +/** Tests ANN payload construction through the vector GlobalIndexer SPI. */ +class PkVectorAnnSegmentFileTest { + + @TempDir java.nio.file.Path tempPath; + + @Test + void testBuildsSingleSourceAnnSegmentWithRowPositionOrdinals() throws Exception { + LocalFileIO fileIO = LocalFileIO.create(); + IndexPathFactory pathFactory = pathFactory(); + Path rawPath = new Path(tempPath.resolve("raw").toUri()); + try (RawVectorSidecarWriter writer = new RawVectorSidecarWriter(fileIO, rawPath, 2)) { + writer.write(new float[] {0, 0}); + writer.write(new float[] {2, 0}); + } + Options options = new Options(); + options.setString("test.vector.dimension", "2"); + options.setString("test.vector.metric", "l2"); + DataField vectorField = new DataField(7, "embedding", DataTypes.ARRAY(DataTypes.FLOAT())); + + IndexFileMeta segment; + try (RawVectorSidecarReader rawReader = new RawVectorSidecarReader(fileIO, rawPath)) { + segment = + new PkVectorAnnSegmentFile(fileIO, pathFactory) + .buildSingleSource( + dataFile("data-1"), + rawReader, + vectorField, + options, + "definition", + "ARRAY", + "l2", + "test-vector-ann", + new byte[] {1, 2}, + 42, + position -> false); + } + + assertThat(segment.indexType()).isEqualTo(PkVectorAnnSegmentFile.PK_VECTOR_ANN); + assertThat(segment.rowCount()).isEqualTo(2); + assertThat(fileIO.exists(pathFactory.toPath(segment))).isTrue(); + PkVectorSegmentMeta metadata = + PkVectorSegmentMeta.deserialize(segment.globalIndexMeta().indexMeta()); + assertThat(metadata.role()).isEqualTo(ANN); + assertThat(metadata.ordinalLayout()).isEqualTo(ROW_POSITION); + assertThat(metadata.sourceFiles()).hasSize(1); + assertThat(metadata.sourceFiles().get(0).fileName()).isEqualTo("data-1"); + assertThat(metadata.liveRowCountAtBuild()).isEqualTo(2); + assertThat(metadata.buildSnapshotId()).isEqualTo(42); + assertThat(metadata.optionsHash()).containsExactly(1, 2); + } + + @Test + void testBuildSkipsNullAndSnapshotDeletedRows() throws Exception { + LocalFileIO fileIO = LocalFileIO.create(); + IndexPathFactory pathFactory = pathFactory(); + Path rawPath = new Path(tempPath.resolve("raw-with-deletes").toUri()); + try (RawVectorSidecarWriter writer = new RawVectorSidecarWriter(fileIO, rawPath, 2)) { + writer.write(new float[] {0, 0}); + writer.write(null); + writer.write(new float[] {2, 0}); + } + Options options = new Options(); + options.setString("test.vector.dimension", "2"); + options.setString("test.vector.metric", "l2"); + DataField vectorField = new DataField(7, "embedding", DataTypes.ARRAY(DataTypes.FLOAT())); + + IndexFileMeta segment; + try (RawVectorSidecarReader rawReader = new RawVectorSidecarReader(fileIO, rawPath)) { + segment = + new PkVectorAnnSegmentFile(fileIO, pathFactory) + .buildSingleSource( + dataFile("data-1", 3), + rawReader, + vectorField, + options, + "definition", + "ARRAY", + "l2", + "test-vector-ann", + new byte[] {1, 2}, + 42, + position -> position == 0); + } + + PkVectorSegmentMeta metadata = + PkVectorSegmentMeta.deserialize(segment.globalIndexMeta().indexMeta()); + assertThat(metadata.liveRowCountAtBuild()).isEqualTo(1); + assertThat(segment.rowCount()).isEqualTo(1); + } + + @Test + void testAnnSearchUsesRowPositionDeletionMask() throws Exception { + LocalFileIO fileIO = LocalFileIO.create(); + IndexPathFactory pathFactory = pathFactory(); + Path rawPath = new Path(tempPath.resolve("ann-search-raw").toUri()); + try (RawVectorSidecarWriter writer = new RawVectorSidecarWriter(fileIO, rawPath, 2)) { + writer.write(new float[] {0, 0}); + writer.write(new float[] {1, 0}); + writer.write(new float[] {2, 0}); + } + Options options = new Options(); + options.setString("test.vector.dimension", "2"); + options.setString("test.vector.metric", "l2"); + DataField vectorField = new DataField(7, "embedding", DataTypes.ARRAY(DataTypes.FLOAT())); + PkVectorAnnSegmentFile annFile = new PkVectorAnnSegmentFile(fileIO, pathFactory); + IndexFileMeta segment; + try (RawVectorSidecarReader rawReader = new RawVectorSidecarReader(fileIO, rawPath)) { + segment = + annFile.buildSingleSource( + dataFile("data-1", 3), + rawReader, + vectorField, + options, + "definition", + "ARRAY", + "l2", + "test-vector-ann", + new byte[] {1, 2}, + 42, + position -> false); + } + PkVectorSegmentMeta metadata = + PkVectorSegmentMeta.deserialize(segment.globalIndexMeta().indexMeta()); + ExecutorService executor = Executors.newSingleThreadExecutor(); + BitmapDeletionVector deletionVector = new BitmapDeletionVector(); + deletionVector.delete(0); + List candidates; + try { + candidates = + new PkVectorAnnSegmentSearcher(fileIO, annFile, vectorField, options, executor) + .search( + segment, + metadata, + new float[] {0, 0}, + 2, + deletionVector, + Collections.emptyMap()); + } finally { + executor.shutdownNow(); + } + + assertThat(candidates) + .extracting(PkVectorAnnSegmentSearcher.Candidate::rowPosition) + .containsExactly(1L, 2L); + assertThat(candidates) + .extracting(PkVectorAnnSegmentSearcher.Candidate::distance) + .containsExactly(1F, 4F); + } + + @Test + void testBuildsAndSearchesMultiSourceAnnSegment() throws Exception { + LocalFileIO fileIO = LocalFileIO.create(); + IndexPathFactory pathFactory = pathFactory(); + Path raw1Path = new Path(tempPath.resolve("multi-raw-1").toUri()); + Path raw2Path = new Path(tempPath.resolve("multi-raw-2").toUri()); + try (RawVectorSidecarWriter writer1 = new RawVectorSidecarWriter(fileIO, raw1Path, 2); + RawVectorSidecarWriter writer2 = new RawVectorSidecarWriter(fileIO, raw2Path, 2)) { + writer1.write(new float[] {5, 0}); + writer1.write(new float[] {10, 0}); + writer2.write(new float[] {0, 0}); + writer2.write(new float[] {2, 0}); + } + Options options = new Options(); + options.setString("test.vector.dimension", "2"); + options.setString("test.vector.metric", "l2"); + DataField vectorField = new DataField(7, "embedding", DataTypes.ARRAY(DataTypes.FLOAT())); + PkVectorAnnSegmentFile annFile = new PkVectorAnnSegmentFile(fileIO, pathFactory); + IndexFileMeta segment; + try (RawVectorSidecarReader raw1 = new RawVectorSidecarReader(fileIO, raw1Path); + RawVectorSidecarReader raw2 = new RawVectorSidecarReader(fileIO, raw2Path)) { + segment = + annFile.buildMultiSource( + Arrays.asList( + new PkVectorAnnSegmentFile.Source(dataFile("data-1"), raw1), + new PkVectorAnnSegmentFile.Source(dataFile("data-2"), raw2)), + vectorField, + options, + "definition", + "ARRAY", + "l2", + "test-vector-ann", + new byte[] {1, 2}, + 42); + } + + PkVectorSegmentMeta metadata = + PkVectorSegmentMeta.deserialize(segment.globalIndexMeta().indexMeta()); + assertThat(metadata.ordinalLayout()) + .isEqualTo(PkVectorSegmentMeta.OrdinalLayout.FILE_POSITION); + assertThat(metadata.sourceFiles()) + .extracting(PkVectorSegmentMeta.SourceFile::fileName) + .containsExactly("data-1", "data-2"); + + BitmapDeletionVector data2Deletes = new BitmapDeletionVector(); + data2Deletes.delete(0); + Map deletionVectors = + new HashMap<>(); + deletionVectors.put("data-2", data2Deletes); + ExecutorService executor = Executors.newSingleThreadExecutor(); + List candidates; + try { + candidates = + new PkVectorAnnSegmentSearcher(fileIO, annFile, vectorField, options, executor) + .search( + segment, + metadata, + new float[] {0, 0}, + 3, + deletionVectors, + Collections.emptyMap()); + } finally { + executor.shutdownNow(); + } + + assertThat(candidates) + .extracting( + PkVectorAnnSegmentSearcher.Candidate::dataFileName, + PkVectorAnnSegmentSearcher.Candidate::rowPosition) + .containsExactly( + org.assertj.core.groups.Tuple.tuple("data-2", 1L), + org.assertj.core.groups.Tuple.tuple("data-1", 0L), + org.assertj.core.groups.Tuple.tuple("data-1", 1L)); + assertThat(candidates.get(0).distance()) + .isCloseTo(4F, org.assertj.core.data.Offset.offset(0.001F)); + assertThat(candidates.get(1).distance()) + .isCloseTo(25F, org.assertj.core.data.Offset.offset(0.001F)); + assertThat(candidates.get(2).distance()) + .isCloseTo(100F, org.assertj.core.data.Offset.offset(0.001F)); + } + + private static DataFileMeta dataFile(String fileName) { + return dataFile(fileName, 2); + } + + private static DataFileMeta dataFile(String fileName, long rowCount) { + return DataFileMeta.forAppend( + fileName, + 100, + rowCount, + SimpleStats.EMPTY_STATS, + 0, + 0, + 1, + Collections.emptyList(), + null, + FileSource.APPEND, + null, + null, + null, + null); + } + + private IndexPathFactory pathFactory() { + Path directory = new Path(tempPath.toUri()); + return new IndexPathFactory() { + @Override + public Path toPath(String fileName) { + return new Path(directory, fileName); + } + + @Override + public Path newPath() { + return new Path(directory, UUID.randomUUID().toString()); + } + + @Override + public boolean isExternalPath() { + return false; + } + }; + } +} diff --git a/paimon-core/src/test/java/org/apache/paimon/index/pkvector/PkVectorSegmentMetaTest.java b/paimon-core/src/test/java/org/apache/paimon/index/pkvector/PkVectorSegmentMetaTest.java new file mode 100644 index 000000000000..afb0f068ed2e --- /dev/null +++ b/paimon-core/src/test/java/org/apache/paimon/index/pkvector/PkVectorSegmentMetaTest.java @@ -0,0 +1,116 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.index.pkvector; + +import org.apache.paimon.io.DataOutputSerializer; + +import org.junit.jupiter.api.Test; + +import java.util.Arrays; + +import static org.apache.paimon.index.pkvector.PkVectorSegmentMeta.OrdinalLayout.FILE_POSITION; +import static org.apache.paimon.index.pkvector.PkVectorSegmentMeta.Role.ANN; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/** Tests for {@link PkVectorSegmentMeta}. */ +class PkVectorSegmentMetaTest { + + @Test + void testRoundTrip() { + PkVectorSegmentMeta metadata = + new PkVectorSegmentMeta( + ANN, + "1d4502f1-9cf0-4d86-8d8d-5cc9ac05e108", + 7, + "VECTOR(1024)", + "l2", + "ivf-pq", + Arrays.asList( + new PkVectorSegmentMeta.SourceFile("data-1", 3, 0, 100, 1024), + new PkVectorSegmentMeta.SourceFile("data-2", 3, 0, 50, 512)), + FILE_POSITION, + 120, + 42, + new byte[] {1, 2, 3}, + new byte[] {4, 5, 6}); + + PkVectorSegmentMeta restored = PkVectorSegmentMeta.deserialize(metadata.serialize()); + + assertThat(restored.role()).isEqualTo(ANN); + assertThat(restored.indexDefinitionId()).isEqualTo(metadata.indexDefinitionId()); + assertThat(restored.vectorFieldId()).isEqualTo(7); + assertThat(restored.vectorTypeFingerprint()).isEqualTo("VECTOR(1024)"); + assertThat(restored.metric()).isEqualTo("l2"); + assertThat(restored.algorithm()).isEqualTo("ivf-pq"); + assertThat(restored.sourceFiles()).isEqualTo(metadata.sourceFiles()); + assertThat(restored.ordinalLayout()).isEqualTo(FILE_POSITION); + assertThat(restored.liveRowCountAtBuild()).isEqualTo(120); + assertThat(restored.buildSnapshotId()).isEqualTo(42); + assertThat(restored.optionsHash()).containsExactly(1, 2, 3); + assertThat(restored.payloadMetadata()).containsExactly(4, 5, 6); + } + + @Test + void testRejectTrailingBytes() { + PkVectorSegmentMeta metadata = + new PkVectorSegmentMeta( + ANN, + "index", + 1, + "VECTOR(2)", + "l2", + "ivf-pq", + Arrays.asList(new PkVectorSegmentMeta.SourceFile("data", 1, 0, 1, 8)), + FILE_POSITION, + 1, + 1, + new byte[0]); + byte[] bytes = Arrays.copyOf(metadata.serialize(), metadata.serialize().length + 1); + + assertThatThrownBy(() -> PkVectorSegmentMeta.deserialize(bytes)) + .hasMessageContaining("Unexpected trailing bytes"); + } + + @Test + void testRejectsPreReleaseLayoutWithoutPayloadMetadata() throws Exception { + DataOutputSerializer output = new DataOutputSerializer(128); + output.writeInt(1); + output.writeByte(ANN.ordinal()); + output.writeUTF("index"); + output.writeInt(7); + output.writeUTF("VECTOR(2)"); + output.writeUTF("l2"); + output.writeUTF("ivf-pq"); + output.writeInt(1); + output.writeUTF("data-1"); + output.writeLong(3); + output.writeInt(0); + output.writeLong(10); + output.writeLong(100); + output.writeByte(FILE_POSITION.ordinal()); + output.writeLong(9); + output.writeLong(42); + output.writeInt(1); + output.writeByte(1); + + assertThatThrownBy(() -> PkVectorSegmentMeta.deserialize(output.getCopyOfBuffer())) + .hasMessageContaining("Failed to deserialize primary-key vector segment metadata"); + } +} diff --git a/paimon-core/src/test/java/org/apache/paimon/index/pkvector/RawVectorSidecarTest.java b/paimon-core/src/test/java/org/apache/paimon/index/pkvector/RawVectorSidecarTest.java new file mode 100644 index 000000000000..883e37e51447 --- /dev/null +++ b/paimon-core/src/test/java/org/apache/paimon/index/pkvector/RawVectorSidecarTest.java @@ -0,0 +1,166 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.index.pkvector; + +import org.apache.paimon.data.GenericArray; +import org.apache.paimon.fs.Path; +import org.apache.paimon.fs.local.LocalFileIO; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/** Tests for raw vector sidecars. */ +class RawVectorSidecarTest { + + @TempDir java.nio.file.Path tempPath; + + @Test + void testRoundTripByRowPosition() throws Exception { + LocalFileIO fileIO = LocalFileIO.create(); + Path path = new Path(tempPath.resolve("raw-vector").toUri()); + try (RawVectorSidecarWriter writer = new RawVectorSidecarWriter(fileIO, path, 3)) { + writer.write(new float[] {1, 2, 3}); + writer.write(null); + writer.write(new GenericArray(new float[] {4, 5, 6})); + } + + try (RawVectorSidecarReader reader = new RawVectorSidecarReader(fileIO, path)) { + assertThat(reader.dimension()).isEqualTo(3); + assertThat(reader.rowCount()).isEqualTo(3); + assertThat(reader.rowPositionForOrdinal(2)).isEqualTo(2); + assertThat(reader.readVector(2)).containsExactly(4, 5, 6); + assertThat(reader.readVector(1)).isNull(); + assertThat(reader.readVector(0)).containsExactly(1, 2, 3); + } + } + + @Test + void testRejectNonFiniteVectorWithoutCorruptingFile() throws Exception { + LocalFileIO fileIO = LocalFileIO.create(); + Path path = new Path(tempPath.resolve("finite-vector").toUri()); + try (RawVectorSidecarWriter writer = new RawVectorSidecarWriter(fileIO, path, 2)) { + assertThatThrownBy(() -> writer.write(new float[] {Float.NaN, 1})) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("must be finite"); + assertThat(writer.rowCount()).isZero(); + writer.write(new float[] {2, 3}); + } + + try (RawVectorSidecarReader reader = new RawVectorSidecarReader(fileIO, path)) { + assertThat(reader.rowCount()).isEqualTo(1); + assertThat(reader.readVector(0)).containsExactly(2, 3); + } + } + + @Test + void testSequentialReadIntoReusableBuffer() throws Exception { + LocalFileIO fileIO = LocalFileIO.create(); + Path path = new Path(tempPath.resolve("sequential-vector").toUri()); + try (RawVectorSidecarWriter writer = new RawVectorSidecarWriter(fileIO, path, 2)) { + writer.write(new float[] {1, 2}); + writer.write(null); + writer.write(new float[] {3, 4}); + } + + try (RawVectorSidecarReader reader = new RawVectorSidecarReader(fileIO, path)) { + float[] buffer = new float[2]; + reader.rewind(); + assertThat(reader.readNextVector(buffer)).isTrue(); + assertThat(buffer).containsExactly(1, 2); + assertThat(reader.readNextVector(buffer)).isFalse(); + assertThat(reader.readNextVector(buffer)).isTrue(); + assertThat(buffer).containsExactly(3, 4); + assertThatThrownBy(() -> reader.readNextVector(buffer)) + .hasMessageContaining("No raw vector remains"); + } + } + + @Test + void testExactSearchSkipsNullAndDeletedPositions() throws Exception { + LocalFileIO fileIO = LocalFileIO.create(); + Path path = new Path(tempPath.resolve("search-vector").toUri()); + try (RawVectorSidecarWriter writer = new RawVectorSidecarWriter(fileIO, path, 2)) { + writer.write(new float[] {0, 0}); + writer.write(null); + writer.write(new float[] {1, 0}); + writer.write(new float[] {0, 2}); + writer.write(new float[] {3, 3}); + } + + try (RawVectorSidecarReader reader = new RawVectorSidecarReader(fileIO, path)) { + List candidates = + reader.search(new float[] {0, 0}, "l2", 2, position -> position == 0); + + assertThat(candidates) + .extracting(RawVectorSidecarReader.Candidate::rowPosition) + .containsExactly(2L, 3L); + assertThat(candidates) + .extracting(RawVectorSidecarReader.Candidate::distance) + .containsExactly(1.0f, 4.0f); + } + } + + @Test + void testCosineSearch() throws Exception { + LocalFileIO fileIO = LocalFileIO.create(); + Path path = new Path(tempPath.resolve("cosine-vector").toUri()); + try (RawVectorSidecarWriter writer = new RawVectorSidecarWriter(fileIO, path, 2)) { + writer.write(new float[] {1, 0}); + writer.write(new float[] {0, 1}); + writer.write(new float[] {2, 0}); + } + + try (RawVectorSidecarReader reader = new RawVectorSidecarReader(fileIO, path)) { + List candidates = + reader.search(new float[] {1, 0}, "cosine", 3, position -> false); + + assertThat(candidates) + .extracting(RawVectorSidecarReader.Candidate::rowPosition) + .containsExactly(0L, 2L, 1L); + assertThat(candidates) + .extracting(RawVectorSidecarReader.Candidate::distance) + .containsExactly(0.0f, 0.0f, 1.0f); + } + } + + @Test + void testInnerProductSearch() throws Exception { + LocalFileIO fileIO = LocalFileIO.create(); + Path path = new Path(tempPath.resolve("inner-product-vector").toUri()); + try (RawVectorSidecarWriter writer = new RawVectorSidecarWriter(fileIO, path, 2)) { + writer.write(new float[] {1, 0}); + writer.write(new float[] {0, 1}); + writer.write(new float[] {2, 0}); + } + + try (RawVectorSidecarReader reader = new RawVectorSidecarReader(fileIO, path)) { + List candidates = + reader.search(new float[] {1, 0}, "inner_product", 3, position -> false); + + assertThat(candidates) + .extracting(RawVectorSidecarReader.Candidate::rowPosition) + .containsExactly(2L, 0L, 1L); + } + } +} From e1ebfe8e0d463257548a7c5ec355d2c0cd1876e7 Mon Sep 17 00:00:00 2001 From: JingsongLi Date: Sat, 11 Jul 2026 08:49:29 +0800 Subject: [PATCH 03/19] [core] Simplify primary-key vector index activation --- docs/generated/core_configuration.html | 66 +++++++++++++++++++ .../java/org/apache/paimon/CoreOptions.java | 56 +--------------- .../pkvector/PkVectorRawSegmentFile.java | 4 +- .../io/KeyValueVectorSidecarWriter.java | 4 +- .../paimon/schema/SchemaValidation.java | 15 ++--- .../PrimaryKeyVectorIndexValidationTest.java | 3 +- 6 files changed, 79 insertions(+), 69 deletions(-) diff --git a/docs/generated/core_configuration.html b/docs/generated/core_configuration.html index 401d451b8693..84f36b8273b7 100644 --- a/docs/generated/core_configuration.html +++ b/docs/generated/core_configuration.html @@ -1205,6 +1205,72 @@ Boolean Enables clustering by non-primary key fields. When set to true, the physical sort order of data files is determined by the configured 'clustering.columns' instead of the primary key, optimizing query performance for non-PK columns. + +
pk-vector.ann.max-rows
+ 100000 + Long + Target maximum rows in one primary-key ANN segment. A single oversized source file is not split. + + +
pk-vector.ann.max-source-files
+ 32 + Integer + Maximum source data files represented by one primary-key ANN segment. + + +
pk-vector.ann.min-rows
+ 10000 + Long + Minimum live rows required before a bucket vector segment is built as ANN. + + +
pk-vector.distance.metric
+ "inner_product" + String + Distance metric persisted by the primary-key vector index. Supported values are l2, cosine, and inner_product. + + +
pk-vector.index.column
+ (none) + String + VECTOR column indexed by the primary-key vector index. + + +
pk-vector.index.name
+ (none) + String + Name of the bucket-local primary-key vector index. + + +
pk-vector.index.options
+ (none) + String + Algorithm-specific options as a JSON object. Unqualified keys are scoped to pk-vector.index.type; fully qualified index or fields.<column> keys are preserved. + + +
pk-vector.index.type
+ (none) + String + Vector index algorithm identifier, for example 'ivf-pq'. The implementation validates it through the vector-index SPI. + + +
pk-vector.l0.max-rows
+ 50000 + Long + Maximum raw vector rows in a bucket before vector minor compaction. + + +
pk-vector.l0.max-segments
+ 8 + Integer + Maximum raw vector segments in a bucket before vector minor compaction. + + +
pk-vector.refine-factor
+ 4 + Integer + Initial ANN candidate multiplier for primary-key vector search. It is not a visibility correctness boundary. +
postpone.batch-write-fixed-bucket
true diff --git a/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java b/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java index 8a8f262bb398..86f9a59e87eb 100644 --- a/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java +++ b/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java @@ -2726,18 +2726,6 @@ public String toString() { "The batch size for lateral vector search. Each batch executes vector " + "topK search and table lookup for multiple query vectors."); - /** - * State of the bucket-local vector index for a primary-key table. The definition is persisted - * in table options so every writer uses the same indexed column and algorithm. - */ - public static final ConfigOption PK_VECTOR_INDEX_STATE = - key("pk-vector.index.state") - .enumType(PrimaryKeyVectorIndexState.class) - .defaultValue(PrimaryKeyVectorIndexState.DISABLED) - .withDescription( - "Lifecycle state of the bucket-local primary-key vector index. " - + "Only BUILDING and ACTIVE cause writers to create vector sidecars."); - public static final ConfigOption PK_VECTOR_INDEX_NAME = key("pk-vector.index.name") .stringType() @@ -4346,18 +4334,10 @@ public int vectorSearchLateralJoinBatchSize() { return options.get(VECTOR_SEARCH_LATERAL_JOIN_BATCH_SIZE); } - public PrimaryKeyVectorIndexState primaryKeyVectorIndexState() { - return options.get(PK_VECTOR_INDEX_STATE); - } - public boolean primaryKeyVectorIndexEnabled() { - return primaryKeyVectorIndexState() != PrimaryKeyVectorIndexState.DISABLED; - } - - public boolean primaryKeyVectorIndexWriteEnabled() { - PrimaryKeyVectorIndexState state = primaryKeyVectorIndexState(); - return state == PrimaryKeyVectorIndexState.BUILDING - || state == PrimaryKeyVectorIndexState.ACTIVE; + return options.getOptional(PK_VECTOR_INDEX_NAME).isPresent() + || options.getOptional(PK_VECTOR_INDEX_COLUMN).isPresent() + || options.getOptional(PK_VECTOR_INDEX_TYPE).isPresent(); } @Nullable @@ -5236,36 +5216,6 @@ public InlineElement getDescription() { } } - /** Lifecycle state of a bucket-local primary-key vector index. */ - public enum PrimaryKeyVectorIndexState implements DescribedEnum { - DISABLED("disabled", "No primary-key vector index is configured."), - BUILDING("building", "Existing files are being backfilled; new files create raw sidecars."), - ACTIVE( - "active", - "The index is available for vector search and new files create raw sidecars."), - DROPPING( - "dropping", - "New sidecars are disabled while existing vector index files are removed."); - - private final String value; - private final String description; - - PrimaryKeyVectorIndexState(String value, String description) { - this.value = value; - this.description = description; - } - - @Override - public String toString() { - return value; - } - - @Override - public InlineElement getDescription() { - return text(description); - } - } - /** Strategy for handling rows whose nested-key contains null values. */ public enum NestedKeyNullStrategy implements DescribedEnum { MERGE( diff --git a/paimon-core/src/main/java/org/apache/paimon/index/pkvector/PkVectorRawSegmentFile.java b/paimon-core/src/main/java/org/apache/paimon/index/pkvector/PkVectorRawSegmentFile.java index 657363a4decd..a8d0b129d26d 100644 --- a/paimon-core/src/main/java/org/apache/paimon/index/pkvector/PkVectorRawSegmentFile.java +++ b/paimon-core/src/main/java/org/apache/paimon/index/pkvector/PkVectorRawSegmentFile.java @@ -18,7 +18,7 @@ package org.apache.paimon.index.pkvector; -import org.apache.paimon.data.InternalVector; +import org.apache.paimon.data.InternalArray; import org.apache.paimon.fs.FileIO; import org.apache.paimon.fs.Path; import org.apache.paimon.index.GlobalIndexMeta; @@ -124,7 +124,7 @@ private Writer( } @Override - public void write(@Nullable InternalVector vector) throws IOException { + public void write(@Nullable InternalArray vector) throws IOException { rawWriter.write(vector); } diff --git a/paimon-core/src/main/java/org/apache/paimon/io/KeyValueVectorSidecarWriter.java b/paimon-core/src/main/java/org/apache/paimon/io/KeyValueVectorSidecarWriter.java index b61adb4666d4..11bd426f21c8 100644 --- a/paimon-core/src/main/java/org/apache/paimon/io/KeyValueVectorSidecarWriter.java +++ b/paimon-core/src/main/java/org/apache/paimon/io/KeyValueVectorSidecarWriter.java @@ -18,7 +18,7 @@ package org.apache.paimon.io; -import org.apache.paimon.data.InternalVector; +import org.apache.paimon.data.InternalArray; import javax.annotation.Nullable; @@ -29,7 +29,7 @@ /** Synchronous vector sidecar owned by one key-value data-file writer. */ public interface KeyValueVectorSidecarWriter extends Closeable { - void write(@Nullable InternalVector vector) throws IOException; + void write(@Nullable InternalArray vector) throws IOException; void complete(DataFileMeta sourceFile) throws IOException; diff --git a/paimon-core/src/main/java/org/apache/paimon/schema/SchemaValidation.java b/paimon-core/src/main/java/org/apache/paimon/schema/SchemaValidation.java index 0b5c930e9996..8cdc0ba76338 100644 --- a/paimon-core/src/main/java/org/apache/paimon/schema/SchemaValidation.java +++ b/paimon-core/src/main/java/org/apache/paimon/schema/SchemaValidation.java @@ -896,24 +896,19 @@ private static void validatePrimaryKeyVectorIndex(TableSchema schema, CoreOption String indexType = options.primaryKeyVectorIndexType(); checkArgument( !StringUtils.isNullOrWhitespaceOnly(indexName), - "pk-vector.index.name must be configured when pk-vector.index.state = %s.", - options.primaryKeyVectorIndexState()); + "pk-vector.index.name must be configured when a primary-key vector index is defined."); checkArgument( !StringUtils.isNullOrWhitespaceOnly(indexColumn), - "pk-vector.index.column must be configured when pk-vector.index.state = %s.", - options.primaryKeyVectorIndexState()); + "pk-vector.index.column must be configured when a primary-key vector index is defined."); checkArgument( !StringUtils.isNullOrWhitespaceOnly(indexType), - "pk-vector.index.type must be configured when pk-vector.index.state = %s.", - options.primaryKeyVectorIndexState()); + "pk-vector.index.type must be configured when a primary-key vector index is defined."); checkArgument( !schema.primaryKeys().isEmpty(), - "pk-vector.index.state = %s requires a primary-key table.", - options.primaryKeyVectorIndexState()); + "Primary-key vector index requires a primary-key table."); checkArgument( options.deletionVectorsEnabled(), - "pk-vector.index.state = %s requires deletion-vectors.enabled = true.", - options.primaryKeyVectorIndexState()); + "Primary-key vector index requires deletion-vectors.enabled = true."); checkArgument( options.mergeEngine() == MergeEngine.DEDUPLICATE || options.mergeEngine() == MergeEngine.PARTIAL_UPDATE, diff --git a/paimon-core/src/test/java/org/apache/paimon/schema/PrimaryKeyVectorIndexValidationTest.java b/paimon-core/src/test/java/org/apache/paimon/schema/PrimaryKeyVectorIndexValidationTest.java index f8277792f14e..4a32e4d2d549 100644 --- a/paimon-core/src/test/java/org/apache/paimon/schema/PrimaryKeyVectorIndexValidationTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/schema/PrimaryKeyVectorIndexValidationTest.java @@ -126,7 +126,7 @@ void testRequiresVectorColumn() { } @Test - void testRequiresCompleteDefinitionOutsideDisabledState() { + void testRequiresCompleteDefinition() { Map options = enabledOptions(); options.remove(CoreOptions.PK_VECTOR_INDEX_NAME.key()); @@ -183,7 +183,6 @@ private static Map enabledOptions() { Map options = new HashMap<>(); options.put(CoreOptions.BUCKET.key(), "1"); options.put(CoreOptions.DELETION_VECTORS_ENABLED.key(), "true"); - options.put(CoreOptions.PK_VECTOR_INDEX_STATE.key(), "building"); options.put(CoreOptions.PK_VECTOR_INDEX_NAME.key(), "embedding_index"); options.put(CoreOptions.PK_VECTOR_INDEX_COLUMN.key(), "embedding"); options.put(CoreOptions.PK_VECTOR_INDEX_TYPE.key(), "ivf-pq"); From bd9e8b78e1b21a0457b998ab3618a6cf01c3de9c Mon Sep 17 00:00:00 2001 From: JingsongLi Date: Sat, 11 Jul 2026 09:00:39 +0800 Subject: [PATCH 04/19] [core] Derive primary-key vector index identity --- docs/generated/core_configuration.html | 6 ------ .../java/org/apache/paimon/CoreOptions.java | 14 +------------ .../PrimaryKeyVectorIndexOptions.java | 20 +++++++++++++++++-- .../paimon/schema/SchemaValidation.java | 4 ---- .../PrimaryKeyVectorIndexOptionsTest.java | 19 ++++++++++++++++++ .../PrimaryKeyVectorIndexValidationTest.java | 8 ++------ 6 files changed, 40 insertions(+), 31 deletions(-) diff --git a/docs/generated/core_configuration.html b/docs/generated/core_configuration.html index 84f36b8273b7..b29313e9f50d 100644 --- a/docs/generated/core_configuration.html +++ b/docs/generated/core_configuration.html @@ -1235,12 +1235,6 @@ String VECTOR column indexed by the primary-key vector index. - -
pk-vector.index.name
- (none) - String - Name of the bucket-local primary-key vector index. -
pk-vector.index.options
(none) diff --git a/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java b/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java index 86f9a59e87eb..08bf8cec9711 100644 --- a/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java +++ b/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java @@ -2726,12 +2726,6 @@ public String toString() { "The batch size for lateral vector search. Each batch executes vector " + "topK search and table lookup for multiple query vectors."); - public static final ConfigOption PK_VECTOR_INDEX_NAME = - key("pk-vector.index.name") - .stringType() - .noDefaultValue() - .withDescription("Name of the bucket-local primary-key vector index."); - public static final ConfigOption PK_VECTOR_INDEX_COLUMN = key("pk-vector.index.column") .stringType() @@ -4335,16 +4329,10 @@ public int vectorSearchLateralJoinBatchSize() { } public boolean primaryKeyVectorIndexEnabled() { - return options.getOptional(PK_VECTOR_INDEX_NAME).isPresent() - || options.getOptional(PK_VECTOR_INDEX_COLUMN).isPresent() + return options.getOptional(PK_VECTOR_INDEX_COLUMN).isPresent() || options.getOptional(PK_VECTOR_INDEX_TYPE).isPresent(); } - @Nullable - public String primaryKeyVectorIndexName() { - return options.get(PK_VECTOR_INDEX_NAME); - } - @Nullable public String primaryKeyVectorIndexColumn() { return options.get(PK_VECTOR_INDEX_COLUMN); diff --git a/paimon-core/src/main/java/org/apache/paimon/index/pkvector/PrimaryKeyVectorIndexOptions.java b/paimon-core/src/main/java/org/apache/paimon/index/pkvector/PrimaryKeyVectorIndexOptions.java index 0cfc1a911cf9..e5c09a572fc3 100644 --- a/paimon-core/src/main/java/org/apache/paimon/index/pkvector/PrimaryKeyVectorIndexOptions.java +++ b/paimon-core/src/main/java/org/apache/paimon/index/pkvector/PrimaryKeyVectorIndexOptions.java @@ -21,6 +21,7 @@ import org.apache.paimon.CoreOptions; import org.apache.paimon.options.Options; import org.apache.paimon.utils.JsonSerdeUtil; +import org.apache.paimon.utils.StringUtils; import java.nio.charset.StandardCharsets; import java.security.MessageDigest; @@ -45,10 +46,25 @@ public static Options resolve(CoreOptions coreOptions) { } public static byte[] hash(CoreOptions coreOptions) { - String canonicalJson = JsonSerdeUtil.toJson(algorithmOptions(coreOptions)); + return sha256(JsonSerdeUtil.toJson(algorithmOptions(coreOptions))); + } + + public static String definitionId( + int vectorFieldId, String vectorTypeFingerprint, CoreOptions coreOptions) { + checkArgument( + vectorTypeFingerprint != null && !vectorTypeFingerprint.trim().isEmpty(), + "Vector type fingerprint must not be empty."); + TreeMap definition = new TreeMap<>(); + definition.put("field-id", Integer.toString(vectorFieldId)); + definition.put("field-type", vectorTypeFingerprint); + definition.putAll(algorithmOptions(coreOptions)); + return StringUtils.byteToHexString(sha256(JsonSerdeUtil.toJson(definition))); + } + + private static byte[] sha256(String value) { try { return MessageDigest.getInstance("SHA-256") - .digest(canonicalJson.getBytes(StandardCharsets.UTF_8)); + .digest(value.getBytes(StandardCharsets.UTF_8)); } catch (NoSuchAlgorithmException e) { throw new IllegalStateException("SHA-256 is not available.", e); } diff --git a/paimon-core/src/main/java/org/apache/paimon/schema/SchemaValidation.java b/paimon-core/src/main/java/org/apache/paimon/schema/SchemaValidation.java index 8cdc0ba76338..075ab8a784cf 100644 --- a/paimon-core/src/main/java/org/apache/paimon/schema/SchemaValidation.java +++ b/paimon-core/src/main/java/org/apache/paimon/schema/SchemaValidation.java @@ -891,12 +891,8 @@ private static void validatePrimaryKeyVectorIndex(TableSchema schema, CoreOption return; } - String indexName = options.primaryKeyVectorIndexName(); String indexColumn = options.primaryKeyVectorIndexColumn(); String indexType = options.primaryKeyVectorIndexType(); - checkArgument( - !StringUtils.isNullOrWhitespaceOnly(indexName), - "pk-vector.index.name must be configured when a primary-key vector index is defined."); checkArgument( !StringUtils.isNullOrWhitespaceOnly(indexColumn), "pk-vector.index.column must be configured when a primary-key vector index is defined."); diff --git a/paimon-core/src/test/java/org/apache/paimon/index/pkvector/PrimaryKeyVectorIndexOptionsTest.java b/paimon-core/src/test/java/org/apache/paimon/index/pkvector/PrimaryKeyVectorIndexOptionsTest.java index c89141aa1521..53295870f7bc 100644 --- a/paimon-core/src/test/java/org/apache/paimon/index/pkvector/PrimaryKeyVectorIndexOptionsTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/index/pkvector/PrimaryKeyVectorIndexOptionsTest.java @@ -61,6 +61,25 @@ void testHashIncludesEffectiveTopLevelAlgorithmOptions() { PrimaryKeyVectorIndexOptions.hash(coreOptions(null, "ivf-pq.nlist", "65"))); } + @Test + void testDefinitionIdIsStableAndDefinitionSensitive() { + CoreOptions first = coreOptions("{\"nlist\":64,\"pq.m\":8}"); + CoreOptions reordered = coreOptions("{\"pq.m\":8,\"nlist\":64}"); + String definitionId = + PrimaryKeyVectorIndexOptions.definitionId(7, "VECTOR", first); + + assertThat(PrimaryKeyVectorIndexOptions.definitionId(7, "VECTOR", reordered)) + .isEqualTo(definitionId); + assertThat(PrimaryKeyVectorIndexOptions.definitionId(8, "VECTOR", first)) + .isNotEqualTo(definitionId); + assertThat(PrimaryKeyVectorIndexOptions.definitionId(7, "VECTOR", first)) + .isNotEqualTo(definitionId); + assertThat( + PrimaryKeyVectorIndexOptions.definitionId( + 7, "VECTOR", coreOptions("{\"nlist\":65,\"pq.m\":8}"))) + .isNotEqualTo(definitionId); + } + @Test void testRejectsNonObjectOptions() { assertThatThrownBy(() -> PrimaryKeyVectorIndexOptions.resolve(coreOptions("[1,2]"))) diff --git a/paimon-core/src/test/java/org/apache/paimon/schema/PrimaryKeyVectorIndexValidationTest.java b/paimon-core/src/test/java/org/apache/paimon/schema/PrimaryKeyVectorIndexValidationTest.java index 4a32e4d2d549..0c2ec3100e62 100644 --- a/paimon-core/src/test/java/org/apache/paimon/schema/PrimaryKeyVectorIndexValidationTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/schema/PrimaryKeyVectorIndexValidationTest.java @@ -126,12 +126,9 @@ void testRequiresVectorColumn() { } @Test - void testRequiresCompleteDefinition() { + void testIndexNameIsNotRequired() { Map options = enabledOptions(); - options.remove(CoreOptions.PK_VECTOR_INDEX_NAME.key()); - - assertThatThrownBy(() -> validateTableSchema(schema(options))) - .hasMessageContaining("pk-vector.index.name must be configured"); + assertThatCode(() -> validateTableSchema(schema(options))).doesNotThrowAnyException(); } @Test @@ -183,7 +180,6 @@ private static Map enabledOptions() { Map options = new HashMap<>(); options.put(CoreOptions.BUCKET.key(), "1"); options.put(CoreOptions.DELETION_VECTORS_ENABLED.key(), "true"); - options.put(CoreOptions.PK_VECTOR_INDEX_NAME.key(), "embedding_index"); options.put(CoreOptions.PK_VECTOR_INDEX_COLUMN.key(), "embedding"); options.put(CoreOptions.PK_VECTOR_INDEX_TYPE.key(), "ivf-pq"); return options; From a517865daad105cd45f6c2e9bbe55b1262904b95 Mon Sep 17 00:00:00 2001 From: JingsongLi Date: Sat, 11 Jul 2026 10:25:36 +0800 Subject: [PATCH 05/19] [core] Scope primary-key vector options by field --- docs/generated/core_configuration.html | 22 +-- .../java/org/apache/paimon/CoreOptions.java | 99 +++++------ .../PrimaryKeyVectorIndexOptions.java | 81 +++++++-- .../paimon/schema/SchemaValidation.java | 46 +++-- .../PrimaryKeyVectorIndexOptionsTest.java | 157 +++++++++++++++++- .../PrimaryKeyVectorIndexValidationTest.java | 55 +++++- 6 files changed, 348 insertions(+), 112 deletions(-) diff --git a/docs/generated/core_configuration.html b/docs/generated/core_configuration.html index b29313e9f50d..5f1311f23315 100644 --- a/docs/generated/core_configuration.html +++ b/docs/generated/core_configuration.html @@ -1224,28 +1224,10 @@ Minimum live rows required before a bucket vector segment is built as ANN. -
pk-vector.distance.metric
- "inner_product" - String - Distance metric persisted by the primary-key vector index. Supported values are l2, cosine, and inner_product. - - -
pk-vector.index.column
- (none) - String - VECTOR column indexed by the primary-key vector index. - - -
pk-vector.index.options
- (none) - String - Algorithm-specific options as a JSON object. Unqualified keys are scoped to pk-vector.index.type; fully qualified index or fields.<column> keys are preserved. - - -
pk-vector.index.type
+
pk-vector.index.columns
(none) String - Vector index algorithm identifier, for example 'ivf-pq'. The implementation validates it through the vector-index SPI. + Comma-separated VECTOR columns indexed by primary-key vector indexes. Each column owns one index and must define fields.<column>.pk-vector.index.type. Index options and distance metric are also field-scoped. Operational defaults can be overridden through fields.<column>.pk-vector.* options. The first release supports exactly one column.
pk-vector.l0.max-rows
diff --git a/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java b/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java index 08bf8cec9711..870c0c534b4d 100644 --- a/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java +++ b/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java @@ -2726,36 +2726,17 @@ public String toString() { "The batch size for lateral vector search. Each batch executes vector " + "topK search and table lookup for multiple query vectors."); - public static final ConfigOption PK_VECTOR_INDEX_COLUMN = - key("pk-vector.index.column") - .stringType() - .noDefaultValue() - .withDescription("VECTOR column indexed by the primary-key vector index."); - - public static final ConfigOption PK_VECTOR_INDEX_TYPE = - key("pk-vector.index.type") + public static final ConfigOption PK_VECTOR_INDEX_COLUMNS = + key("pk-vector.index.columns") .stringType() .noDefaultValue() .withDescription( - "Vector index algorithm identifier, for example 'ivf-pq'. " - + "The implementation validates it through the vector-index SPI."); - - public static final ConfigOption PK_VECTOR_INDEX_OPTIONS = - key("pk-vector.index.options") - .stringType() - .noDefaultValue() - .withDescription( - "Algorithm-specific options as a JSON object. Unqualified keys are " - + "scoped to pk-vector.index.type; fully qualified index or " - + "fields. keys are preserved."); - - public static final ConfigOption PK_VECTOR_DISTANCE_METRIC = - key("pk-vector.distance.metric") - .stringType() - .defaultValue("inner_product") - .withDescription( - "Distance metric persisted by the primary-key vector index. " - + "Supported values are l2, cosine, and inner_product."); + "Comma-separated VECTOR columns indexed by primary-key vector indexes. " + + "Each column owns one index and must define " + + "fields..pk-vector.index.type. Index options and distance " + + "metric are also field-scoped. Operational defaults can be " + + "overridden through fields..pk-vector.* options. The first " + + "release supports exactly one column."); public static final ConfigOption PK_VECTOR_L0_MAX_SEGMENTS = key("pk-vector.l0.max-segments") @@ -4329,51 +4310,71 @@ public int vectorSearchLateralJoinBatchSize() { } public boolean primaryKeyVectorIndexEnabled() { - return options.getOptional(PK_VECTOR_INDEX_COLUMN).isPresent() - || options.getOptional(PK_VECTOR_INDEX_TYPE).isPresent(); + return options.getOptional(PK_VECTOR_INDEX_COLUMNS).isPresent(); + } + + public List primaryKeyVectorIndexColumns() { + String columns = options.get(PK_VECTOR_INDEX_COLUMNS); + if (columns == null) { + return Collections.emptyList(); + } + return Arrays.stream(columns.split(",", -1)).map(String::trim).collect(Collectors.toList()); } @Nullable - public String primaryKeyVectorIndexColumn() { - return options.get(PK_VECTOR_INDEX_COLUMN); + public String primaryKeyVectorIndexType(String column) { + return options.get("fields." + column + ".pk-vector.index.type"); } @Nullable - public String primaryKeyVectorIndexType() { - return options.get(PK_VECTOR_INDEX_TYPE); + public String primaryKeyVectorIndexOptions(String column) { + return options.get("fields." + column + ".pk-vector.index.options"); + } + + public String primaryKeyVectorDistanceMetric(String column) { + String metric = options.get("fields." + column + ".pk-vector.distance.metric"); + return (metric == null ? "inner_product" : metric) + .toLowerCase(Locale.ROOT) + .replace('-', '_'); } @Nullable - public String primaryKeyVectorIndexOptions() { - return options.get(PK_VECTOR_INDEX_OPTIONS); + private String primaryKeyVectorFieldOption(String column, ConfigOption option) { + return options.get("fields." + column + "." + option.key()); } - public String primaryKeyVectorDistanceMetric() { - return options.get(PK_VECTOR_DISTANCE_METRIC).toLowerCase(Locale.ROOT).replace('-', '_'); + private T primaryKeyVectorOption(String column, ConfigOption option) { + String fieldValue = primaryKeyVectorFieldOption(column, option); + if (fieldValue == null) { + return options.get(option); + } + Options fieldOptions = new Options(); + fieldOptions.setString(option.key(), fieldValue); + return fieldOptions.get(option); } - public int primaryKeyVectorL0MaxSegments() { - return options.get(PK_VECTOR_L0_MAX_SEGMENTS); + public int primaryKeyVectorL0MaxSegments(String column) { + return primaryKeyVectorOption(column, PK_VECTOR_L0_MAX_SEGMENTS); } - public long primaryKeyVectorL0MaxRows() { - return options.get(PK_VECTOR_L0_MAX_ROWS); + public long primaryKeyVectorL0MaxRows(String column) { + return primaryKeyVectorOption(column, PK_VECTOR_L0_MAX_ROWS); } - public long primaryKeyVectorAnnMinRows() { - return options.get(PK_VECTOR_ANN_MIN_ROWS); + public long primaryKeyVectorAnnMinRows(String column) { + return primaryKeyVectorOption(column, PK_VECTOR_ANN_MIN_ROWS); } - public long primaryKeyVectorAnnMaxRows() { - return options.get(PK_VECTOR_ANN_MAX_ROWS); + public long primaryKeyVectorAnnMaxRows(String column) { + return primaryKeyVectorOption(column, PK_VECTOR_ANN_MAX_ROWS); } - public int primaryKeyVectorAnnMaxSourceFiles() { - return options.get(PK_VECTOR_ANN_MAX_SOURCE_FILES); + public int primaryKeyVectorAnnMaxSourceFiles(String column) { + return primaryKeyVectorOption(column, PK_VECTOR_ANN_MAX_SOURCE_FILES); } - public int primaryKeyVectorRefineFactor() { - return options.get(PK_VECTOR_REFINE_FACTOR); + public int primaryKeyVectorRefineFactor(String column) { + return primaryKeyVectorOption(column, PK_VECTOR_REFINE_FACTOR); } /** Specifies the merge engine for table with primary key. */ diff --git a/paimon-core/src/main/java/org/apache/paimon/index/pkvector/PrimaryKeyVectorIndexOptions.java b/paimon-core/src/main/java/org/apache/paimon/index/pkvector/PrimaryKeyVectorIndexOptions.java index e5c09a572fc3..d27a1588b593 100644 --- a/paimon-core/src/main/java/org/apache/paimon/index/pkvector/PrimaryKeyVectorIndexOptions.java +++ b/paimon-core/src/main/java/org/apache/paimon/index/pkvector/PrimaryKeyVectorIndexOptions.java @@ -27,6 +27,7 @@ import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; import java.util.LinkedHashMap; +import java.util.List; import java.util.Map; import java.util.TreeMap; @@ -38,29 +39,55 @@ public final class PrimaryKeyVectorIndexOptions { private PrimaryKeyVectorIndexOptions() {} public static Options resolve(CoreOptions coreOptions) { + return resolve(coreOptions, singleColumn(coreOptions)); + } + + public static Options resolve(CoreOptions coreOptions, String field) { Options resolved = new Options(coreOptions.toConfiguration().toMap()); - for (Map.Entry option : algorithmOptions(coreOptions).entrySet()) { + for (Map.Entry option : algorithmOptions(coreOptions, field).entrySet()) { resolved.setString(option.getKey(), option.getValue()); } return resolved; } public static byte[] hash(CoreOptions coreOptions) { - return sha256(JsonSerdeUtil.toJson(algorithmOptions(coreOptions))); + return hash(coreOptions, singleColumn(coreOptions)); + } + + public static byte[] hash(CoreOptions coreOptions, String field) { + return sha256(JsonSerdeUtil.toJson(fingerprintOptions(coreOptions, field))); } public static String definitionId( int vectorFieldId, String vectorTypeFingerprint, CoreOptions coreOptions) { + return definitionId( + vectorFieldId, vectorTypeFingerprint, coreOptions, singleColumn(coreOptions)); + } + + public static String definitionId( + int vectorFieldId, + String vectorTypeFingerprint, + CoreOptions coreOptions, + String field) { checkArgument( vectorTypeFingerprint != null && !vectorTypeFingerprint.trim().isEmpty(), "Vector type fingerprint must not be empty."); TreeMap definition = new TreeMap<>(); definition.put("field-id", Integer.toString(vectorFieldId)); definition.put("field-type", vectorTypeFingerprint); - definition.putAll(algorithmOptions(coreOptions)); + definition.putAll(fingerprintOptions(coreOptions, field)); return StringUtils.byteToHexString(sha256(JsonSerdeUtil.toJson(definition))); } + public static String singleColumn(CoreOptions coreOptions) { + List columns = coreOptions.primaryKeyVectorIndexColumns(); + checkArgument( + columns.size() == 1, + "pk-vector.index.columns must contain exactly one column in the first release, but is %s.", + columns); + return columns.get(0); + } + private static byte[] sha256(String value) { try { return MessageDigest.getInstance("SHA-256") @@ -70,40 +97,44 @@ private static byte[] sha256(String value) { } } - private static Map algorithmOptions(CoreOptions coreOptions) { - String algorithm = coreOptions.primaryKeyVectorIndexType(); + private static Map algorithmOptions(CoreOptions coreOptions, String field) { + String indexTypeKey = "fields." + field + ".pk-vector.index.type"; + String indexOptionsKey = "fields." + field + ".pk-vector.index.options"; + String algorithm = coreOptions.primaryKeyVectorIndexType(field); checkArgument( algorithm != null && !algorithm.trim().isEmpty(), - "pk-vector.index.type must be configured before resolving index options."); + "%s must be configured before resolving index options.", + indexTypeKey); TreeMap options = new TreeMap<>(); - String field = coreOptions.primaryKeyVectorIndexColumn(); String algorithmPrefix = algorithm + "."; - String fieldPrefix = field == null ? null : "fields." + field + "."; + String fieldPrefix = "fields." + field + "."; for (Map.Entry entry : coreOptions.toConfiguration().toMap().entrySet()) { if (entry.getKey().startsWith(algorithmPrefix) - || (fieldPrefix != null && entry.getKey().startsWith(fieldPrefix))) { + || (entry.getKey().startsWith(fieldPrefix) + && !entry.getKey().startsWith(fieldPrefix + "pk-vector."))) { options.put(entry.getKey(), entry.getValue()); } } - String serialized = coreOptions.primaryKeyVectorIndexOptions(); + String serialized = coreOptions.primaryKeyVectorIndexOptions(field); if (serialized != null && !serialized.trim().isEmpty()) { LinkedHashMap parsed; try { parsed = JsonSerdeUtil.parseJsonMap(serialized, String.class); } catch (RuntimeException e) { throw new IllegalArgumentException( - "pk-vector.index.options must be a JSON object of option key-value pairs.", - e); + indexOptionsKey + " must be a JSON object of option key-value pairs.", e); } for (Map.Entry entry : parsed.entrySet()) { String key = entry.getKey(); String value = entry.getValue(); checkArgument( key != null && !key.trim().isEmpty(), - "pk-vector.index.options contains an empty option key."); + "%s contains an empty option key.", + indexOptionsKey); checkArgument( value != null, - "pk-vector.index.options value for key %s must not be null.", + "%s value for key %s must not be null.", + indexOptionsKey, key); String qualifiedKey = key.startsWith(algorithmPrefix) || key.startsWith("fields.") @@ -112,11 +143,29 @@ private static Map algorithmOptions(CoreOptions coreOptions) { String previous = options.put(qualifiedKey, value); checkArgument( previous == null || previous.equals(value), - "pk-vector.index.options defines conflicting values for %s.", + "%s defines conflicting values for %s.", + indexOptionsKey, qualifiedKey); } } - options.put(algorithmPrefix + "metric", coreOptions.primaryKeyVectorDistanceMetric()); + options.put(algorithmPrefix + "metric", coreOptions.primaryKeyVectorDistanceMetric(field)); return options; } + + private static Map fingerprintOptions(CoreOptions coreOptions, String field) { + String fieldPrefix = "fields." + field + "."; + TreeMap fingerprint = new TreeMap<>(); + Map options = algorithmOptions(coreOptions, field); + for (Map.Entry entry : options.entrySet()) { + if (!entry.getKey().startsWith(fieldPrefix)) { + fingerprint.put(entry.getKey(), entry.getValue()); + } + } + for (Map.Entry entry : options.entrySet()) { + if (entry.getKey().startsWith(fieldPrefix)) { + fingerprint.put(entry.getKey().substring(fieldPrefix.length()), entry.getValue()); + } + } + return fingerprint; + } } diff --git a/paimon-core/src/main/java/org/apache/paimon/schema/SchemaValidation.java b/paimon-core/src/main/java/org/apache/paimon/schema/SchemaValidation.java index 075ab8a784cf..23c58015caa3 100644 --- a/paimon-core/src/main/java/org/apache/paimon/schema/SchemaValidation.java +++ b/paimon-core/src/main/java/org/apache/paimon/schema/SchemaValidation.java @@ -891,14 +891,24 @@ private static void validatePrimaryKeyVectorIndex(TableSchema schema, CoreOption return; } - String indexColumn = options.primaryKeyVectorIndexColumn(); - String indexType = options.primaryKeyVectorIndexType(); + List indexColumns = options.primaryKeyVectorIndexColumns(); + checkArgument( + new HashSet<>(indexColumns).size() == indexColumns.size(), + "pk-vector.index.columns must not contain duplicate columns, but is %s.", + indexColumns); + checkArgument( + indexColumns.size() == 1, + "pk-vector.index.columns must contain exactly one column in the first release, but is %s.", + indexColumns); + String indexColumn = indexColumns.get(0); + String indexType = options.primaryKeyVectorIndexType(indexColumn); checkArgument( !StringUtils.isNullOrWhitespaceOnly(indexColumn), - "pk-vector.index.column must be configured when a primary-key vector index is defined."); + "pk-vector.index.columns must contain a non-empty column."); checkArgument( !StringUtils.isNullOrWhitespaceOnly(indexType), - "pk-vector.index.type must be configured when a primary-key vector index is defined."); + "fields.%s.pk-vector.index.type must be configured when a primary-key vector index is defined.", + indexColumn); checkArgument( !schema.primaryKeys().isEmpty(), "Primary-key vector index requires a primary-key table."); @@ -921,7 +931,7 @@ private static void validatePrimaryKeyVectorIndex(TableSchema schema, CoreOption checkArgument( !options.pkClusteringOverride(), "Primary-key vector index does not support pk-clustering-override."); - PrimaryKeyVectorIndexOptions.resolve(options); + PrimaryKeyVectorIndexOptions.resolve(options, indexColumn); DataField vectorField = schema.fields().stream() @@ -930,38 +940,40 @@ private static void validatePrimaryKeyVectorIndex(TableSchema schema, CoreOption .orElse(null); checkArgument( vectorField != null && vectorField.type().getTypeRoot() == VECTOR, - "pk-vector.index.column '%s' must reference a VECTOR column.", + "pk-vector.index.columns entry '%s' must reference a VECTOR column.", indexColumn); checkArgument( ((VectorType) vectorField.type()).getElementType().getTypeRoot() == DataTypeRoot.FLOAT, - "pk-vector.index.column '%s' must use FLOAT elements.", + "pk-vector.index.columns entry '%s' must use FLOAT elements.", indexColumn); checkArgument( Arrays.asList("l2", "cosine", "inner_product") - .contains(options.primaryKeyVectorDistanceMetric()), - "pk-vector.distance.metric must be one of l2, cosine, inner_product, but is %s.", - options.primaryKeyVectorDistanceMetric()); + .contains(options.primaryKeyVectorDistanceMetric(indexColumn)), + "fields.%s.pk-vector.distance.metric must be one of l2, cosine, inner_product, but is %s.", + indexColumn, + options.primaryKeyVectorDistanceMetric(indexColumn)); checkArgument( - options.primaryKeyVectorL0MaxSegments() > 0, + options.primaryKeyVectorL0MaxSegments(indexColumn) > 0, "pk-vector.l0.max-segments must be greater than 0."); checkArgument( - options.primaryKeyVectorL0MaxRows() > 0, + options.primaryKeyVectorL0MaxRows(indexColumn) > 0, "pk-vector.l0.max-rows must be greater than 0."); checkArgument( - options.primaryKeyVectorAnnMinRows() > 0, + options.primaryKeyVectorAnnMinRows(indexColumn) > 0, "pk-vector.ann.min-rows must be greater than 0."); checkArgument( - options.primaryKeyVectorAnnMaxRows() > 0, + options.primaryKeyVectorAnnMaxRows(indexColumn) > 0, "pk-vector.ann.max-rows must be greater than 0."); checkArgument( - options.primaryKeyVectorAnnMaxRows() >= options.primaryKeyVectorAnnMinRows(), + options.primaryKeyVectorAnnMaxRows(indexColumn) + >= options.primaryKeyVectorAnnMinRows(indexColumn), "pk-vector.ann.max-rows must be greater than or equal to pk-vector.ann.min-rows."); checkArgument( - options.primaryKeyVectorAnnMaxSourceFiles() > 0, + options.primaryKeyVectorAnnMaxSourceFiles(indexColumn) > 0, "pk-vector.ann.max-source-files must be greater than 0."); checkArgument( - options.primaryKeyVectorRefineFactor() > 0, + options.primaryKeyVectorRefineFactor(indexColumn) > 0, "pk-vector.refine-factor must be greater than 0."); } diff --git a/paimon-core/src/test/java/org/apache/paimon/index/pkvector/PrimaryKeyVectorIndexOptionsTest.java b/paimon-core/src/test/java/org/apache/paimon/index/pkvector/PrimaryKeyVectorIndexOptionsTest.java index 53295870f7bc..e32a8561e7ca 100644 --- a/paimon-core/src/test/java/org/apache/paimon/index/pkvector/PrimaryKeyVectorIndexOptionsTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/index/pkvector/PrimaryKeyVectorIndexOptionsTest.java @@ -32,6 +32,96 @@ /** Tests for {@link PrimaryKeyVectorIndexOptions}. */ class PrimaryKeyVectorIndexOptionsTest { + @Test + void testPluralFieldRegistryEnablesIndex() { + Map options = new HashMap<>(); + options.put(CoreOptions.PK_VECTOR_INDEX_COLUMNS.key(), "embedding"); + + assertThat(new CoreOptions(options).primaryKeyVectorIndexEnabled()).isTrue(); + } + + @Test + void testFieldRegistryIsTheOnlyEnableSwitch() { + Map options = new HashMap<>(); + options.put("pk-vector.index.column", "embedding"); + options.put("pk-vector.index.type", "ivf-pq"); + + assertThat(new CoreOptions(options).primaryKeyVectorIndexEnabled()).isFalse(); + } + + @Test + void testIndexTypeMustBeFieldScoped() { + Map options = new HashMap<>(); + options.put(CoreOptions.PK_VECTOR_INDEX_COLUMNS.key(), "embedding"); + options.put("pk-vector.index.type", "ivf-pq"); + + assertThat(new CoreOptions(options).primaryKeyVectorIndexType("embedding")).isNull(); + } + + @Test + void testIndexOptionsMustBeFieldScoped() { + Map options = new HashMap<>(); + options.put(CoreOptions.PK_VECTOR_INDEX_COLUMNS.key(), "embedding"); + options.put("pk-vector.index.options", "{\"nlist\":64}"); + + assertThat(new CoreOptions(options).primaryKeyVectorIndexOptions("embedding")).isNull(); + } + + @Test + void testDistanceMetricMustBeFieldScoped() { + Map options = new HashMap<>(); + options.put(CoreOptions.PK_VECTOR_INDEX_COLUMNS.key(), "embedding"); + options.put("pk-vector.distance.metric", "l2"); + + assertThat(new CoreOptions(options).primaryKeyVectorDistanceMetric("embedding")) + .isEqualTo("inner_product"); + } + + @Test + void testFieldScopedDistanceMetricOverridesTableDefault() { + Map options = new HashMap<>(); + options.put(CoreOptions.PK_VECTOR_INDEX_COLUMNS.key(), "embedding"); + options.put("pk-vector.distance.metric", "l2"); + options.put("fields.embedding.pk-vector.distance.metric", "cosine"); + + assertThat(new CoreOptions(options).primaryKeyVectorDistanceMetric("embedding")) + .isEqualTo("cosine"); + } + + @Test + void testFieldScopedAnnThresholdOverridesTableDefault() { + Map options = new HashMap<>(); + options.put(CoreOptions.PK_VECTOR_INDEX_COLUMNS.key(), "embedding"); + options.put(CoreOptions.PK_VECTOR_ANN_MIN_ROWS.key(), "10000"); + options.put("fields.embedding.pk-vector.ann.min-rows", "20000"); + options.put("fields.embedding.pk-vector.l0.max-segments", "4"); + options.put("fields.embedding.pk-vector.l0.max-rows", "30000"); + options.put("fields.embedding.pk-vector.ann.max-rows", "90000"); + options.put("fields.embedding.pk-vector.ann.max-source-files", "16"); + options.put("fields.embedding.pk-vector.refine-factor", "6"); + + CoreOptions coreOptions = new CoreOptions(options); + assertThat(coreOptions.primaryKeyVectorAnnMinRows("embedding")).isEqualTo(20_000L); + assertThat(coreOptions.primaryKeyVectorL0MaxSegments("embedding")).isEqualTo(4); + assertThat(coreOptions.primaryKeyVectorL0MaxRows("embedding")).isEqualTo(30_000L); + assertThat(coreOptions.primaryKeyVectorAnnMaxRows("embedding")).isEqualTo(90_000L); + assertThat(coreOptions.primaryKeyVectorAnnMaxSourceFiles("embedding")).isEqualTo(16); + assertThat(coreOptions.primaryKeyVectorRefineFactor("embedding")).isEqualTo(6); + } + + @Test + void testFieldScopedJsonOptionsOverrideTableDefault() { + Map options = new HashMap<>(); + options.put(CoreOptions.PK_VECTOR_INDEX_COLUMNS.key(), "embedding"); + options.put("fields.embedding.pk-vector.index.type", "ivf-pq"); + options.put("pk-vector.index.options", "{\"nlist\":64}"); + options.put("fields.embedding.pk-vector.index.options", "{\"nlist\":128}"); + + Options resolved = PrimaryKeyVectorIndexOptions.resolve(new CoreOptions(options)); + + assertThat(resolved.get("ivf-pq.nlist")).isEqualTo("128"); + } + @Test void testResolvesShortAndQualifiedAlgorithmOptions() { CoreOptions coreOptions = @@ -80,6 +170,61 @@ void testDefinitionIdIsStableAndDefinitionSensitive() { .isNotEqualTo(definitionId); } + @Test + void testDefinitionIdExcludesOperationalThresholds() { + CoreOptions first = coreOptions("{\"nlist\":64}"); + first.toConfiguration().setString("fields.embedding.pk-vector.ann.min-rows", "10000"); + CoreOptions second = coreOptions("{\"nlist\":64}"); + second.toConfiguration().setString("fields.embedding.pk-vector.ann.min-rows", "20000"); + + assertThat(PrimaryKeyVectorIndexOptions.definitionId(7, "VECTOR", first)) + .isEqualTo( + PrimaryKeyVectorIndexOptions.definitionId(7, "VECTOR", second)); + } + + @Test + void testDefinitionIdIsStableAcrossFieldRename() { + Map firstOptions = new HashMap<>(); + firstOptions.put(CoreOptions.PK_VECTOR_INDEX_COLUMNS.key(), "embedding"); + firstOptions.put("fields.embedding.pk-vector.index.type", "ivf-pq"); + firstOptions.put("fields.embedding.ivf-pq.nlist", "64"); + Map renamedOptions = new HashMap<>(); + renamedOptions.put(CoreOptions.PK_VECTOR_INDEX_COLUMNS.key(), "renamed_embedding"); + renamedOptions.put("fields.renamed_embedding.pk-vector.index.type", "ivf-pq"); + renamedOptions.put("fields.renamed_embedding.ivf-pq.nlist", "64"); + + assertThat( + PrimaryKeyVectorIndexOptions.definitionId( + 7, "VECTOR", new CoreOptions(firstOptions), "embedding")) + .isEqualTo( + PrimaryKeyVectorIndexOptions.definitionId( + 7, + "VECTOR", + new CoreOptions(renamedOptions), + "renamed_embedding")); + } + + @Test + void testDefinitionIdIgnoresShadowedTableDefault() { + Map firstOptions = new HashMap<>(); + firstOptions.put(CoreOptions.PK_VECTOR_INDEX_COLUMNS.key(), "embedding"); + firstOptions.put("fields.embedding.pk-vector.index.type", "ivf-pq"); + firstOptions.put("ivf-pq.nlist", "64"); + firstOptions.put("fields.embedding.ivf-pq.nlist", "128"); + Map changedDefault = new HashMap<>(firstOptions); + changedDefault.put("ivf-pq.nlist", "96"); + + assertThat( + PrimaryKeyVectorIndexOptions.definitionId( + 7, "VECTOR", new CoreOptions(firstOptions), "embedding")) + .isEqualTo( + PrimaryKeyVectorIndexOptions.definitionId( + 7, + "VECTOR", + new CoreOptions(changedDefault), + "embedding")); + } + @Test void testRejectsNonObjectOptions() { assertThatThrownBy(() -> PrimaryKeyVectorIndexOptions.resolve(coreOptions("[1,2]"))) @@ -92,8 +237,8 @@ void testRejectsNonObjectOptions() { void testAnnBuildBoundsDefaults() { CoreOptions options = coreOptions(null); - assertThat(options.primaryKeyVectorAnnMaxRows()).isEqualTo(100_000L); - assertThat(options.primaryKeyVectorAnnMaxSourceFiles()).isEqualTo(32); + assertThat(options.primaryKeyVectorAnnMaxRows("embedding")).isEqualTo(100_000L); + assertThat(options.primaryKeyVectorAnnMaxSourceFiles("embedding")).isEqualTo(32); } private static CoreOptions coreOptions(String indexOptions) { @@ -103,11 +248,11 @@ private static CoreOptions coreOptions(String indexOptions) { private static CoreOptions coreOptions( String indexOptions, String additionalKey, String additionalValue) { Map options = new HashMap<>(); - options.put(CoreOptions.PK_VECTOR_INDEX_TYPE.key(), "ivf-pq"); - options.put(CoreOptions.PK_VECTOR_INDEX_COLUMN.key(), "embedding"); - options.put(CoreOptions.PK_VECTOR_DISTANCE_METRIC.key(), "l2"); + options.put("fields.embedding.pk-vector.index.type", "ivf-pq"); + options.put(CoreOptions.PK_VECTOR_INDEX_COLUMNS.key(), "embedding"); + options.put("fields.embedding.pk-vector.distance.metric", "l2"); if (indexOptions != null) { - options.put(CoreOptions.PK_VECTOR_INDEX_OPTIONS.key(), indexOptions); + options.put("fields.embedding.pk-vector.index.options", indexOptions); } if (additionalKey != null) { options.put(additionalKey, additionalValue); diff --git a/paimon-core/src/test/java/org/apache/paimon/schema/PrimaryKeyVectorIndexValidationTest.java b/paimon-core/src/test/java/org/apache/paimon/schema/PrimaryKeyVectorIndexValidationTest.java index 0c2ec3100e62..e00fd7a825be 100644 --- a/paimon-core/src/test/java/org/apache/paimon/schema/PrimaryKeyVectorIndexValidationTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/schema/PrimaryKeyVectorIndexValidationTest.java @@ -36,12 +36,58 @@ /** Tests for primary-key vector index option validation. */ class PrimaryKeyVectorIndexValidationTest { + @Test + void testValidPluralPrimaryKeyVectorIndexConfiguration() { + Map options = new HashMap<>(); + options.put(CoreOptions.BUCKET.key(), "1"); + options.put(CoreOptions.DELETION_VECTORS_ENABLED.key(), "true"); + options.put(CoreOptions.PK_VECTOR_INDEX_COLUMNS.key(), " embedding "); + options.put("fields.embedding.pk-vector.index.type", "ivf-pq"); + + assertThatCode(() -> validateTableSchema(schema(options))).doesNotThrowAnyException(); + } + + @Test + void testRejectsMultiplePrimaryKeyVectorIndexColumnsForFirstRelease() { + Map options = new HashMap<>(); + options.put(CoreOptions.BUCKET.key(), "1"); + options.put(CoreOptions.DELETION_VECTORS_ENABLED.key(), "true"); + options.put(CoreOptions.PK_VECTOR_INDEX_COLUMNS.key(), "embedding,other_embedding"); + options.put("fields.embedding.pk-vector.index.type", "ivf-pq"); + + assertThatThrownBy(() -> validateTableSchema(schema(options))) + .hasMessageContaining("exactly one column") + .hasMessageContaining("pk-vector.index.columns"); + } + + @Test + void testRejectsDuplicatePrimaryKeyVectorIndexColumns() { + Map options = new HashMap<>(); + options.put(CoreOptions.BUCKET.key(), "1"); + options.put(CoreOptions.DELETION_VECTORS_ENABLED.key(), "true"); + options.put(CoreOptions.PK_VECTOR_INDEX_COLUMNS.key(), "embedding,embedding"); + + assertThatThrownBy(() -> validateTableSchema(schema(options))) + .hasMessageContaining("pk-vector.index.columns") + .hasMessageContaining("duplicate"); + } + @Test void testValidPrimaryKeyVectorIndex() { assertThatCode(() -> validateTableSchema(schema(enabledOptions()))) .doesNotThrowAnyException(); } + @Test + void testRequiresFieldScopedIndexType() { + Map options = enabledOptions(); + options.remove("fields.embedding.pk-vector.index.type"); + options.put("pk-vector.index.type", "ivf-pq"); + + assertThatThrownBy(() -> validateTableSchema(schema(options))) + .hasMessageContaining("fields.embedding.pk-vector.index.type"); + } + @Test void testRequiresPrimaryKeyTable() { Map options = enabledOptions(); @@ -119,7 +165,8 @@ void testRejectsPkClusteringOverride() { @Test void testRequiresVectorColumn() { Map options = enabledOptions(); - options.put(CoreOptions.PK_VECTOR_INDEX_COLUMN.key(), "payload"); + options.put(CoreOptions.PK_VECTOR_INDEX_COLUMNS.key(), "payload"); + options.put("fields.payload.pk-vector.index.type", "ivf-pq"); assertThatThrownBy(() -> validateTableSchema(schema(options))) .hasMessageContaining("must reference a VECTOR column"); @@ -153,7 +200,7 @@ void testRequiresFloatVectorElements() { @Test void testRejectsUnsupportedDistanceMetric() { Map options = enabledOptions(); - options.put(CoreOptions.PK_VECTOR_DISTANCE_METRIC.key(), "manhattan"); + options.put("fields.embedding.pk-vector.distance.metric", "manhattan"); assertThatThrownBy(() -> validateTableSchema(schema(options))) .hasMessageContaining("pk-vector.distance.metric") @@ -180,8 +227,8 @@ private static Map enabledOptions() { Map options = new HashMap<>(); options.put(CoreOptions.BUCKET.key(), "1"); options.put(CoreOptions.DELETION_VECTORS_ENABLED.key(), "true"); - options.put(CoreOptions.PK_VECTOR_INDEX_COLUMN.key(), "embedding"); - options.put(CoreOptions.PK_VECTOR_INDEX_TYPE.key(), "ivf-pq"); + options.put(CoreOptions.PK_VECTOR_INDEX_COLUMNS.key(), "embedding"); + options.put("fields.embedding.pk-vector.index.type", "ivf-pq"); return options; } From f4d4f1acf58b79bf0dfc915411efe444fd621cf7 Mon Sep 17 00:00:00 2001 From: JingsongLi Date: Sat, 11 Jul 2026 17:52:47 +0800 Subject: [PATCH 06/19] [core] Rename vector options with columns --- .../apache/paimon/schema/SchemaManager.java | 35 +++++++++++ .../paimon/schema/SchemaManagerTest.java | 61 +++++++++++++++++++ 2 files changed, 96 insertions(+) diff --git a/paimon-core/src/main/java/org/apache/paimon/schema/SchemaManager.java b/paimon-core/src/main/java/org/apache/paimon/schema/SchemaManager.java index 0791c2539e47..45ae08583b93 100644 --- a/paimon-core/src/main/java/org/apache/paimon/schema/SchemaManager.java +++ b/paimon-core/src/main/java/org/apache/paimon/schema/SchemaManager.java @@ -833,6 +833,21 @@ private static Map applyRenameColumnsToOptions( newOptions.put(SEQUENCE_FIELD.key(), String.join(",", newSequenceFields)); } + // primary-key vector index column rename + String vectorIndexColumnsStr = options.get(CoreOptions.PK_VECTOR_INDEX_COLUMNS.key()); + Set vectorIndexColumns = Collections.emptySet(); + if (!StringUtils.isNullOrWhitespaceOnly(vectorIndexColumnsStr)) { + List vectorColumns = + Arrays.stream(vectorIndexColumnsStr.split(",")) + .map(String::trim) + .collect(Collectors.toList()); + vectorIndexColumns = new HashSet<>(vectorColumns); + List newVectorColumns = + applyNotNestedColumnRename(vectorColumns, renameMappings); + newOptions.put( + CoreOptions.PK_VECTOR_INDEX_COLUMNS.key(), String.join(",", newVectorColumns)); + } + // case 2: the option key is composed of certain fixed prefixes, suffixes, and the field // name, while the option value doesn't contain field names. List> fieldNameToOptionKeys = @@ -894,6 +909,26 @@ private static Map applyRenameColumnsToOptions( } } + // A vector field owns both pk-vector options and algorithm-specific options below its + // fields.. namespace. Move the complete namespace after the specialized option + // rewrites above so keys handled by case 3 are not processed twice. + for (RenameColumn rename : renameColumns) { + String fieldName = rename.fieldNames()[0]; + if (!vectorIndexColumns.contains(fieldName)) { + continue; + } + String oldPrefix = FIELDS_PREFIX + "." + fieldName + "."; + String newPrefix = FIELDS_PREFIX + "." + rename.newName() + "."; + for (String key : options.keySet()) { + if (key.startsWith(oldPrefix)) { + String value = newOptions.remove(key); + if (value != null) { + newOptions.put(newPrefix + key.substring(oldPrefix.length()), value); + } + } + } + } + return newOptions; } diff --git a/paimon-core/src/test/java/org/apache/paimon/schema/SchemaManagerTest.java b/paimon-core/src/test/java/org/apache/paimon/schema/SchemaManagerTest.java index 59a4f90f6105..cccf0b9299fc 100644 --- a/paimon-core/src/test/java/org/apache/paimon/schema/SchemaManagerTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/schema/SchemaManagerTest.java @@ -26,6 +26,7 @@ import org.apache.paimon.fs.FileIOFinder; import org.apache.paimon.fs.Path; import org.apache.paimon.fs.local.LocalFileIO; +import org.apache.paimon.index.pkvector.PrimaryKeyVectorIndexOptions; import org.apache.paimon.reader.RecordReaderIterator; import org.apache.paimon.table.FileStoreTable; import org.apache.paimon.table.FileStoreTableFactory; @@ -170,6 +171,66 @@ public void testUpdateOptions() throws Exception { assertThat(latest.get().options()).containsEntry("new_k", "new_v"); } + @Test + public void testRenamePrimaryKeyVectorIndexColumnOptions() throws Exception { + Map options = new HashMap<>(); + options.put(CoreOptions.BUCKET.key(), "1"); + options.put(CoreOptions.DELETION_VECTORS_ENABLED.key(), "true"); + options.put(CoreOptions.PK_VECTOR_INDEX_COLUMNS.key(), "embedding"); + options.put("fields.embedding.pk-vector.index.type", "ivf-pq"); + options.put("fields.embedding.pk-vector.distance.metric", "cosine"); + options.put("fields.embedding.pk-vector.index.options", "{\"nlist\":64,\"pq.m\":8}"); + options.put("fields.embedding.pk-vector.ann.min-rows", "20000"); + options.put("fields.embedding.ivf-pq.nprobe", "16"); + Schema schema = + new Schema( + Arrays.asList( + new DataField(0, "id", DataTypes.INT().notNull()), + new DataField( + 1, "embedding", DataTypes.VECTOR(8, DataTypes.FLOAT()))), + Collections.emptyList(), + Collections.singletonList("id"), + options, + ""); + SchemaManager manager = new SchemaManager(LocalFileIO.create(), path); + TableSchema before = manager.createTable(schema); + DataField beforeVector = before.fields().get(1); + String beforeDefinitionId = + PrimaryKeyVectorIndexOptions.definitionId( + beforeVector.id(), + beforeVector.type().asSQLString(), + new CoreOptions(before.options()), + beforeVector.name()); + + manager.commitChanges( + SchemaChange.renameColumn(new String[] {"embedding"}, "renamed_embedding")); + + TableSchema after = manager.latest().get(); + assertThat(after.options()) + .containsEntry(CoreOptions.PK_VECTOR_INDEX_COLUMNS.key(), "renamed_embedding") + .containsEntry("fields.renamed_embedding.pk-vector.index.type", "ivf-pq") + .containsEntry("fields.renamed_embedding.pk-vector.distance.metric", "cosine") + .containsEntry( + "fields.renamed_embedding.pk-vector.index.options", + "{\"nlist\":64,\"pq.m\":8}") + .containsEntry("fields.renamed_embedding.pk-vector.ann.min-rows", "20000") + .containsEntry("fields.renamed_embedding.ivf-pq.nprobe", "16") + .doesNotContainKeys( + "fields.embedding.pk-vector.index.type", + "fields.embedding.pk-vector.distance.metric", + "fields.embedding.pk-vector.index.options", + "fields.embedding.pk-vector.ann.min-rows", + "fields.embedding.ivf-pq.nprobe"); + DataField afterVector = after.fields().get(1); + assertThat( + PrimaryKeyVectorIndexOptions.definitionId( + afterVector.id(), + afterVector.type().asSQLString(), + new CoreOptions(after.options()), + afterVector.name())) + .isEqualTo(beforeDefinitionId); + } + @Test public void testResetSequenceGroupForAggregateFunction() throws Exception { Map options = new HashMap<>(); From fc023ca963402555e3be68b8d82262024cf914bb Mon Sep 17 00:00:00 2001 From: JingsongLi Date: Sat, 11 Jul 2026 18:09:50 +0800 Subject: [PATCH 07/19] [core] Simplify primary-key vector ANN options --- docs/generated/core_configuration.html | 32 +--------- .../java/org/apache/paimon/CoreOptions.java | 62 +------------------ .../paimon/schema/SchemaValidation.java | 19 ------ .../PrimaryKeyVectorIndexOptionsTest.java | 18 ------ .../PrimaryKeyVectorIndexValidationTest.java | 18 ++---- 5 files changed, 10 insertions(+), 139 deletions(-) diff --git a/docs/generated/core_configuration.html b/docs/generated/core_configuration.html index 5f1311f23315..0412807fd277 100644 --- a/docs/generated/core_configuration.html +++ b/docs/generated/core_configuration.html @@ -1205,18 +1205,6 @@ Boolean Enables clustering by non-primary key fields. When set to true, the physical sort order of data files is determined by the configured 'clustering.columns' instead of the primary key, optimizing query performance for non-PK columns. - -
pk-vector.ann.max-rows
- 100000 - Long - Target maximum rows in one primary-key ANN segment. A single oversized source file is not split. - - -
pk-vector.ann.max-source-files
- 32 - Integer - Maximum source data files represented by one primary-key ANN segment. -
pk-vector.ann.min-rows
10000 @@ -1227,25 +1215,7 @@
pk-vector.index.columns
(none) String - Comma-separated VECTOR columns indexed by primary-key vector indexes. Each column owns one index and must define fields.<column>.pk-vector.index.type. Index options and distance metric are also field-scoped. Operational defaults can be overridden through fields.<column>.pk-vector.* options. The first release supports exactly one column. - - -
pk-vector.l0.max-rows
- 50000 - Long - Maximum raw vector rows in a bucket before vector minor compaction. - - -
pk-vector.l0.max-segments
- 8 - Integer - Maximum raw vector segments in a bucket before vector minor compaction. - - -
pk-vector.refine-factor
- 4 - Integer - Initial ANN candidate multiplier for primary-key vector search. It is not a visibility correctness boundary. + Comma-separated VECTOR columns indexed by primary-key vector indexes. Each column owns one index and must define fields.<column>.pk-vector.index.type. Index options and distance metric are also field-scoped. The ANN minimum can be overridden through fields.<column>.pk-vector.ann.min-rows. The first release supports exactly one column.
postpone.batch-write-fixed-bucket
diff --git a/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java b/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java index 870c0c534b4d..17825da46cc2 100644 --- a/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java +++ b/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java @@ -2734,23 +2734,9 @@ public String toString() { "Comma-separated VECTOR columns indexed by primary-key vector indexes. " + "Each column owns one index and must define " + "fields..pk-vector.index.type. Index options and distance " - + "metric are also field-scoped. Operational defaults can be " - + "overridden through fields..pk-vector.* options. The first " - + "release supports exactly one column."); - - public static final ConfigOption PK_VECTOR_L0_MAX_SEGMENTS = - key("pk-vector.l0.max-segments") - .intType() - .defaultValue(8) - .withDescription( - "Maximum raw vector segments in a bucket before vector minor compaction."); - - public static final ConfigOption PK_VECTOR_L0_MAX_ROWS = - key("pk-vector.l0.max-rows") - .longType() - .defaultValue(50_000L) - .withDescription( - "Maximum raw vector rows in a bucket before vector minor compaction."); + + "metric are also field-scoped. The ANN minimum can be overridden " + + "through fields..pk-vector.ann.min-rows. The first release " + + "supports exactly one column."); public static final ConfigOption PK_VECTOR_ANN_MIN_ROWS = key("pk-vector.ann.min-rows") @@ -2759,28 +2745,6 @@ public String toString() { .withDescription( "Minimum live rows required before a bucket vector segment is built as ANN."); - public static final ConfigOption PK_VECTOR_ANN_MAX_ROWS = - key("pk-vector.ann.max-rows") - .longType() - .defaultValue(100_000L) - .withDescription( - "Target maximum rows in one primary-key ANN segment. A single oversized source file is not split."); - - public static final ConfigOption PK_VECTOR_ANN_MAX_SOURCE_FILES = - key("pk-vector.ann.max-source-files") - .intType() - .defaultValue(32) - .withDescription( - "Maximum source data files represented by one primary-key ANN segment."); - - public static final ConfigOption PK_VECTOR_REFINE_FACTOR = - key("pk-vector.refine-factor") - .intType() - .defaultValue(4) - .withDescription( - "Initial ANN candidate multiplier for primary-key vector search. " - + "It is not a visibility correctness boundary."); - @Immutable public static final ConfigOption PK_CLUSTERING_OVERRIDE = key("pk-clustering-override") @@ -4353,30 +4317,10 @@ private T primaryKeyVectorOption(String column, ConfigOption option) { return fieldOptions.get(option); } - public int primaryKeyVectorL0MaxSegments(String column) { - return primaryKeyVectorOption(column, PK_VECTOR_L0_MAX_SEGMENTS); - } - - public long primaryKeyVectorL0MaxRows(String column) { - return primaryKeyVectorOption(column, PK_VECTOR_L0_MAX_ROWS); - } - public long primaryKeyVectorAnnMinRows(String column) { return primaryKeyVectorOption(column, PK_VECTOR_ANN_MIN_ROWS); } - public long primaryKeyVectorAnnMaxRows(String column) { - return primaryKeyVectorOption(column, PK_VECTOR_ANN_MAX_ROWS); - } - - public int primaryKeyVectorAnnMaxSourceFiles(String column) { - return primaryKeyVectorOption(column, PK_VECTOR_ANN_MAX_SOURCE_FILES); - } - - public int primaryKeyVectorRefineFactor(String column) { - return primaryKeyVectorOption(column, PK_VECTOR_REFINE_FACTOR); - } - /** Specifies the merge engine for table with primary key. */ public enum MergeEngine implements DescribedEnum { DEDUPLICATE("deduplicate", "De-duplicate and keep the last row."), diff --git a/paimon-core/src/main/java/org/apache/paimon/schema/SchemaValidation.java b/paimon-core/src/main/java/org/apache/paimon/schema/SchemaValidation.java index 23c58015caa3..7ce227ddecca 100644 --- a/paimon-core/src/main/java/org/apache/paimon/schema/SchemaValidation.java +++ b/paimon-core/src/main/java/org/apache/paimon/schema/SchemaValidation.java @@ -953,28 +953,9 @@ private static void validatePrimaryKeyVectorIndex(TableSchema schema, CoreOption "fields.%s.pk-vector.distance.metric must be one of l2, cosine, inner_product, but is %s.", indexColumn, options.primaryKeyVectorDistanceMetric(indexColumn)); - checkArgument( - options.primaryKeyVectorL0MaxSegments(indexColumn) > 0, - "pk-vector.l0.max-segments must be greater than 0."); - checkArgument( - options.primaryKeyVectorL0MaxRows(indexColumn) > 0, - "pk-vector.l0.max-rows must be greater than 0."); checkArgument( options.primaryKeyVectorAnnMinRows(indexColumn) > 0, "pk-vector.ann.min-rows must be greater than 0."); - checkArgument( - options.primaryKeyVectorAnnMaxRows(indexColumn) > 0, - "pk-vector.ann.max-rows must be greater than 0."); - checkArgument( - options.primaryKeyVectorAnnMaxRows(indexColumn) - >= options.primaryKeyVectorAnnMinRows(indexColumn), - "pk-vector.ann.max-rows must be greater than or equal to pk-vector.ann.min-rows."); - checkArgument( - options.primaryKeyVectorAnnMaxSourceFiles(indexColumn) > 0, - "pk-vector.ann.max-source-files must be greater than 0."); - checkArgument( - options.primaryKeyVectorRefineFactor(indexColumn) > 0, - "pk-vector.refine-factor must be greater than 0."); } private static void validateSequenceField(TableSchema schema, CoreOptions options) { diff --git a/paimon-core/src/test/java/org/apache/paimon/index/pkvector/PrimaryKeyVectorIndexOptionsTest.java b/paimon-core/src/test/java/org/apache/paimon/index/pkvector/PrimaryKeyVectorIndexOptionsTest.java index e32a8561e7ca..9eaeb7432596 100644 --- a/paimon-core/src/test/java/org/apache/paimon/index/pkvector/PrimaryKeyVectorIndexOptionsTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/index/pkvector/PrimaryKeyVectorIndexOptionsTest.java @@ -94,19 +94,9 @@ void testFieldScopedAnnThresholdOverridesTableDefault() { options.put(CoreOptions.PK_VECTOR_INDEX_COLUMNS.key(), "embedding"); options.put(CoreOptions.PK_VECTOR_ANN_MIN_ROWS.key(), "10000"); options.put("fields.embedding.pk-vector.ann.min-rows", "20000"); - options.put("fields.embedding.pk-vector.l0.max-segments", "4"); - options.put("fields.embedding.pk-vector.l0.max-rows", "30000"); - options.put("fields.embedding.pk-vector.ann.max-rows", "90000"); - options.put("fields.embedding.pk-vector.ann.max-source-files", "16"); - options.put("fields.embedding.pk-vector.refine-factor", "6"); CoreOptions coreOptions = new CoreOptions(options); assertThat(coreOptions.primaryKeyVectorAnnMinRows("embedding")).isEqualTo(20_000L); - assertThat(coreOptions.primaryKeyVectorL0MaxSegments("embedding")).isEqualTo(4); - assertThat(coreOptions.primaryKeyVectorL0MaxRows("embedding")).isEqualTo(30_000L); - assertThat(coreOptions.primaryKeyVectorAnnMaxRows("embedding")).isEqualTo(90_000L); - assertThat(coreOptions.primaryKeyVectorAnnMaxSourceFiles("embedding")).isEqualTo(16); - assertThat(coreOptions.primaryKeyVectorRefineFactor("embedding")).isEqualTo(6); } @Test @@ -233,14 +223,6 @@ void testRejectsNonObjectOptions() { .hasMessageContaining("JSON object"); } - @Test - void testAnnBuildBoundsDefaults() { - CoreOptions options = coreOptions(null); - - assertThat(options.primaryKeyVectorAnnMaxRows("embedding")).isEqualTo(100_000L); - assertThat(options.primaryKeyVectorAnnMaxSourceFiles("embedding")).isEqualTo(32); - } - private static CoreOptions coreOptions(String indexOptions) { return coreOptions(indexOptions, null, null); } diff --git a/paimon-core/src/test/java/org/apache/paimon/schema/PrimaryKeyVectorIndexValidationTest.java b/paimon-core/src/test/java/org/apache/paimon/schema/PrimaryKeyVectorIndexValidationTest.java index e00fd7a825be..dc9f18acf2a2 100644 --- a/paimon-core/src/test/java/org/apache/paimon/schema/PrimaryKeyVectorIndexValidationTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/schema/PrimaryKeyVectorIndexValidationTest.java @@ -208,18 +208,12 @@ void testRejectsUnsupportedDistanceMetric() { } @Test - void testRejectsInvalidAnnBuildBounds() { - Map invalidRowOptions = enabledOptions(); - invalidRowOptions.put(CoreOptions.PK_VECTOR_ANN_MIN_ROWS.key(), "100"); - invalidRowOptions.put(CoreOptions.PK_VECTOR_ANN_MAX_ROWS.key(), "99"); - assertThatThrownBy(() -> validateTableSchema(schema(invalidRowOptions))) - .hasMessageContaining("pk-vector.ann.max-rows") - .hasMessageContaining("greater than or equal"); - - Map invalidSourceFileOptions = enabledOptions(); - invalidSourceFileOptions.put(CoreOptions.PK_VECTOR_ANN_MAX_SOURCE_FILES.key(), "0"); - assertThatThrownBy(() -> validateTableSchema(schema(invalidSourceFileOptions))) - .hasMessageContaining("pk-vector.ann.max-source-files") + void testRejectsInvalidAnnMinimumRows() { + Map options = enabledOptions(); + options.put(CoreOptions.PK_VECTOR_ANN_MIN_ROWS.key(), "0"); + + assertThatThrownBy(() -> validateTableSchema(schema(options))) + .hasMessageContaining("pk-vector.ann.min-rows") .hasMessageContaining("greater than 0"); } From f1d0979ed915768e9254dd81068acc040a65cfa3 Mon Sep 17 00:00:00 2001 From: JingsongLi Date: Sat, 11 Jul 2026 18:15:20 +0800 Subject: [PATCH 08/19] [core] Disallow renaming primary-key vector columns --- .../apache/paimon/schema/SchemaManager.java | 49 ++++++------------ .../PrimaryKeyVectorIndexOptionsTest.java | 22 -------- .../paimon/schema/SchemaManagerTest.java | 50 ++++--------------- 3 files changed, 23 insertions(+), 98 deletions(-) diff --git a/paimon-core/src/main/java/org/apache/paimon/schema/SchemaManager.java b/paimon-core/src/main/java/org/apache/paimon/schema/SchemaManager.java index 45ae08583b93..06f6a1e07768 100644 --- a/paimon-core/src/main/java/org/apache/paimon/schema/SchemaManager.java +++ b/paimon-core/src/main/java/org/apache/paimon/schema/SchemaManager.java @@ -431,6 +431,7 @@ protected void updateLastColumn( } else if (change instanceof RenameColumn) { RenameColumn rename = (RenameColumn) change; assertNotUpdatingPartitionKeys(oldTableSchema, rename.fieldNames(), "rename"); + assertNotRenamingPrimaryKeyVectorIndexColumn(oldTableSchema, rename.fieldNames()); assertNotRenamingBlobColumn(newFields, rename.fieldNames()); new NestedColumnModifier(rename.fieldNames(), lazyIdentifier) { @Override @@ -833,21 +834,6 @@ private static Map applyRenameColumnsToOptions( newOptions.put(SEQUENCE_FIELD.key(), String.join(",", newSequenceFields)); } - // primary-key vector index column rename - String vectorIndexColumnsStr = options.get(CoreOptions.PK_VECTOR_INDEX_COLUMNS.key()); - Set vectorIndexColumns = Collections.emptySet(); - if (!StringUtils.isNullOrWhitespaceOnly(vectorIndexColumnsStr)) { - List vectorColumns = - Arrays.stream(vectorIndexColumnsStr.split(",")) - .map(String::trim) - .collect(Collectors.toList()); - vectorIndexColumns = new HashSet<>(vectorColumns); - List newVectorColumns = - applyNotNestedColumnRename(vectorColumns, renameMappings); - newOptions.put( - CoreOptions.PK_VECTOR_INDEX_COLUMNS.key(), String.join(",", newVectorColumns)); - } - // case 2: the option key is composed of certain fixed prefixes, suffixes, and the field // name, while the option value doesn't contain field names. List> fieldNameToOptionKeys = @@ -909,26 +895,6 @@ private static Map applyRenameColumnsToOptions( } } - // A vector field owns both pk-vector options and algorithm-specific options below its - // fields.. namespace. Move the complete namespace after the specialized option - // rewrites above so keys handled by case 3 are not processed twice. - for (RenameColumn rename : renameColumns) { - String fieldName = rename.fieldNames()[0]; - if (!vectorIndexColumns.contains(fieldName)) { - continue; - } - String oldPrefix = FIELDS_PREFIX + "." + fieldName + "."; - String newPrefix = FIELDS_PREFIX + "." + rename.newName() + "."; - for (String key : options.keySet()) { - if (key.startsWith(oldPrefix)) { - String value = newOptions.remove(key); - if (value != null) { - newOptions.put(newPrefix + key.substring(oldPrefix.length()), value); - } - } - } - } - return newOptions; } @@ -1010,6 +976,19 @@ private static void assertNotRenamingBlobColumn(List fields, String[] } } + private static void assertNotRenamingPrimaryKeyVectorIndexColumn( + TableSchema schema, String[] fieldNames) { + if (fieldNames.length > 1) { + return; + } + String fieldName = fieldNames[0]; + if (new CoreOptions(schema.options()).primaryKeyVectorIndexColumns().contains(fieldName)) { + throw new UnsupportedOperationException( + String.format( + "Cannot rename primary-key vector index column: [%s]", fieldName)); + } + } + private static void assertNotChangingBlobColumnType( List fields, String[] fieldNames, DataType newType) { if (fieldNames.length > 1) { diff --git a/paimon-core/src/test/java/org/apache/paimon/index/pkvector/PrimaryKeyVectorIndexOptionsTest.java b/paimon-core/src/test/java/org/apache/paimon/index/pkvector/PrimaryKeyVectorIndexOptionsTest.java index 9eaeb7432596..5b015f188ee1 100644 --- a/paimon-core/src/test/java/org/apache/paimon/index/pkvector/PrimaryKeyVectorIndexOptionsTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/index/pkvector/PrimaryKeyVectorIndexOptionsTest.java @@ -172,28 +172,6 @@ void testDefinitionIdExcludesOperationalThresholds() { PrimaryKeyVectorIndexOptions.definitionId(7, "VECTOR", second)); } - @Test - void testDefinitionIdIsStableAcrossFieldRename() { - Map firstOptions = new HashMap<>(); - firstOptions.put(CoreOptions.PK_VECTOR_INDEX_COLUMNS.key(), "embedding"); - firstOptions.put("fields.embedding.pk-vector.index.type", "ivf-pq"); - firstOptions.put("fields.embedding.ivf-pq.nlist", "64"); - Map renamedOptions = new HashMap<>(); - renamedOptions.put(CoreOptions.PK_VECTOR_INDEX_COLUMNS.key(), "renamed_embedding"); - renamedOptions.put("fields.renamed_embedding.pk-vector.index.type", "ivf-pq"); - renamedOptions.put("fields.renamed_embedding.ivf-pq.nlist", "64"); - - assertThat( - PrimaryKeyVectorIndexOptions.definitionId( - 7, "VECTOR", new CoreOptions(firstOptions), "embedding")) - .isEqualTo( - PrimaryKeyVectorIndexOptions.definitionId( - 7, - "VECTOR", - new CoreOptions(renamedOptions), - "renamed_embedding")); - } - @Test void testDefinitionIdIgnoresShadowedTableDefault() { Map firstOptions = new HashMap<>(); diff --git a/paimon-core/src/test/java/org/apache/paimon/schema/SchemaManagerTest.java b/paimon-core/src/test/java/org/apache/paimon/schema/SchemaManagerTest.java index cccf0b9299fc..b982ff1df1d3 100644 --- a/paimon-core/src/test/java/org/apache/paimon/schema/SchemaManagerTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/schema/SchemaManagerTest.java @@ -26,7 +26,6 @@ import org.apache.paimon.fs.FileIOFinder; import org.apache.paimon.fs.Path; import org.apache.paimon.fs.local.LocalFileIO; -import org.apache.paimon.index.pkvector.PrimaryKeyVectorIndexOptions; import org.apache.paimon.reader.RecordReaderIterator; import org.apache.paimon.table.FileStoreTable; import org.apache.paimon.table.FileStoreTableFactory; @@ -172,16 +171,12 @@ public void testUpdateOptions() throws Exception { } @Test - public void testRenamePrimaryKeyVectorIndexColumnOptions() throws Exception { + public void testRejectRenamePrimaryKeyVectorIndexColumn() throws Exception { Map options = new HashMap<>(); options.put(CoreOptions.BUCKET.key(), "1"); options.put(CoreOptions.DELETION_VECTORS_ENABLED.key(), "true"); options.put(CoreOptions.PK_VECTOR_INDEX_COLUMNS.key(), "embedding"); options.put("fields.embedding.pk-vector.index.type", "ivf-pq"); - options.put("fields.embedding.pk-vector.distance.metric", "cosine"); - options.put("fields.embedding.pk-vector.index.options", "{\"nlist\":64,\"pq.m\":8}"); - options.put("fields.embedding.pk-vector.ann.min-rows", "20000"); - options.put("fields.embedding.ivf-pq.nprobe", "16"); Schema schema = new Schema( Arrays.asList( @@ -193,42 +188,15 @@ public void testRenamePrimaryKeyVectorIndexColumnOptions() throws Exception { options, ""); SchemaManager manager = new SchemaManager(LocalFileIO.create(), path); - TableSchema before = manager.createTable(schema); - DataField beforeVector = before.fields().get(1); - String beforeDefinitionId = - PrimaryKeyVectorIndexOptions.definitionId( - beforeVector.id(), - beforeVector.type().asSQLString(), - new CoreOptions(before.options()), - beforeVector.name()); + manager.createTable(schema); - manager.commitChanges( - SchemaChange.renameColumn(new String[] {"embedding"}, "renamed_embedding")); - - TableSchema after = manager.latest().get(); - assertThat(after.options()) - .containsEntry(CoreOptions.PK_VECTOR_INDEX_COLUMNS.key(), "renamed_embedding") - .containsEntry("fields.renamed_embedding.pk-vector.index.type", "ivf-pq") - .containsEntry("fields.renamed_embedding.pk-vector.distance.metric", "cosine") - .containsEntry( - "fields.renamed_embedding.pk-vector.index.options", - "{\"nlist\":64,\"pq.m\":8}") - .containsEntry("fields.renamed_embedding.pk-vector.ann.min-rows", "20000") - .containsEntry("fields.renamed_embedding.ivf-pq.nprobe", "16") - .doesNotContainKeys( - "fields.embedding.pk-vector.index.type", - "fields.embedding.pk-vector.distance.metric", - "fields.embedding.pk-vector.index.options", - "fields.embedding.pk-vector.ann.min-rows", - "fields.embedding.ivf-pq.nprobe"); - DataField afterVector = after.fields().get(1); - assertThat( - PrimaryKeyVectorIndexOptions.definitionId( - afterVector.id(), - afterVector.type().asSQLString(), - new CoreOptions(after.options()), - afterVector.name())) - .isEqualTo(beforeDefinitionId); + assertThatThrownBy( + () -> + manager.commitChanges( + SchemaChange.renameColumn( + new String[] {"embedding"}, "renamed_embedding"))) + .isInstanceOf(UnsupportedOperationException.class) + .hasMessage("Cannot rename primary-key vector index column: [embedding]"); } @Test From 8cf83a6fc1bd57fe07b731624915384a165497da Mon Sep 17 00:00:00 2001 From: JingsongLi Date: Sat, 11 Jul 2026 18:19:20 +0800 Subject: [PATCH 09/19] [core] Simplify ANN segment build API --- .../pkvector/PkVectorAnnSegmentFile.java | 82 +------------------ .../pkvector/PkVectorAnnSegmentFileTest.java | 39 +++++---- 2 files changed, 24 insertions(+), 97 deletions(-) diff --git a/paimon-core/src/main/java/org/apache/paimon/index/pkvector/PkVectorAnnSegmentFile.java b/paimon-core/src/main/java/org/apache/paimon/index/pkvector/PkVectorAnnSegmentFile.java index 5c07c5dc035e..e899fc07d812 100644 --- a/paimon-core/src/main/java/org/apache/paimon/index/pkvector/PkVectorAnnSegmentFile.java +++ b/paimon-core/src/main/java/org/apache/paimon/index/pkvector/PkVectorAnnSegmentFile.java @@ -41,15 +41,12 @@ import java.io.IOException; import java.util.ArrayList; -import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Locale; import java.util.Map; import java.util.function.LongPredicate; -import static org.apache.paimon.index.pkvector.PkVectorSegmentMeta.OrdinalLayout.FILE_POSITION; -import static org.apache.paimon.index.pkvector.PkVectorSegmentMeta.OrdinalLayout.ROW_POSITION; import static org.apache.paimon.index.pkvector.PkVectorSegmentMeta.Role.ANN; import static org.apache.paimon.utils.Preconditions.checkArgument; @@ -62,84 +59,7 @@ public PkVectorAnnSegmentFile(FileIO fileIO, IndexPathFactory pathFactory) { super(fileIO, pathFactory); } - public IndexFileMeta buildSingleSource( - DataFileMeta sourceFile, - RawVectorSidecarReader rawVectors, - DataField vectorField, - Options indexOptions, - String indexDefinitionId, - String vectorTypeFingerprint, - String metric, - String algorithm, - byte[] optionsHash, - long buildSnapshotId, - LongPredicate excludedPosition) - throws IOException { - return build( - Collections.singletonList(new Source(sourceFile, rawVectors, excludedPosition)), - ROW_POSITION, - vectorField, - indexOptions, - indexDefinitionId, - vectorTypeFingerprint, - metric, - algorithm, - optionsHash, - buildSnapshotId); - } - - public IndexFileMeta buildMultiSource( - List sources, - DataField vectorField, - Options indexOptions, - String indexDefinitionId, - String vectorTypeFingerprint, - String metric, - String algorithm, - byte[] optionsHash, - long buildSnapshotId) - throws IOException { - checkArgument( - sources.size() > 1, - "A multi-source ANN segment must reference at least two source files."); - return build( - sources, - FILE_POSITION, - vectorField, - indexOptions, - indexDefinitionId, - vectorTypeFingerprint, - metric, - algorithm, - optionsHash, - buildSnapshotId); - } - - IndexFileMeta buildSources( - List sources, - DataField vectorField, - Options indexOptions, - String indexDefinitionId, - String vectorTypeFingerprint, - String metric, - String algorithm, - byte[] optionsHash, - long buildSnapshotId) - throws IOException { - return build( - sources, - sources.size() == 1 ? ROW_POSITION : FILE_POSITION, - vectorField, - indexOptions, - indexDefinitionId, - vectorTypeFingerprint, - metric, - algorithm, - optionsHash, - buildSnapshotId); - } - - private IndexFileMeta build( + public IndexFileMeta build( List sources, PkVectorSegmentMeta.OrdinalLayout ordinalLayout, DataField vectorField, diff --git a/paimon-core/src/test/java/org/apache/paimon/index/pkvector/PkVectorAnnSegmentFileTest.java b/paimon-core/src/test/java/org/apache/paimon/index/pkvector/PkVectorAnnSegmentFileTest.java index fe1546467ca9..64c1e93b0f11 100644 --- a/paimon-core/src/test/java/org/apache/paimon/index/pkvector/PkVectorAnnSegmentFileTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/index/pkvector/PkVectorAnnSegmentFileTest.java @@ -42,6 +42,7 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import static org.apache.paimon.index.pkvector.PkVectorSegmentMeta.OrdinalLayout.FILE_POSITION; import static org.apache.paimon.index.pkvector.PkVectorSegmentMeta.OrdinalLayout.ROW_POSITION; import static org.apache.paimon.index.pkvector.PkVectorSegmentMeta.Role.ANN; import static org.assertj.core.api.Assertions.assertThat; @@ -69,9 +70,11 @@ void testBuildsSingleSourceAnnSegmentWithRowPositionOrdinals() throws Exception try (RawVectorSidecarReader rawReader = new RawVectorSidecarReader(fileIO, rawPath)) { segment = new PkVectorAnnSegmentFile(fileIO, pathFactory) - .buildSingleSource( - dataFile("data-1"), - rawReader, + .build( + Collections.singletonList( + new PkVectorAnnSegmentFile.Source( + dataFile("data-1"), rawReader)), + ROW_POSITION, vectorField, options, "definition", @@ -79,8 +82,7 @@ void testBuildsSingleSourceAnnSegmentWithRowPositionOrdinals() throws Exception "l2", "test-vector-ann", new byte[] {1, 2}, - 42, - position -> false); + 42); } assertThat(segment.indexType()).isEqualTo(PkVectorAnnSegmentFile.PK_VECTOR_ANN); @@ -116,9 +118,13 @@ void testBuildSkipsNullAndSnapshotDeletedRows() throws Exception { try (RawVectorSidecarReader rawReader = new RawVectorSidecarReader(fileIO, rawPath)) { segment = new PkVectorAnnSegmentFile(fileIO, pathFactory) - .buildSingleSource( - dataFile("data-1", 3), - rawReader, + .build( + Collections.singletonList( + new PkVectorAnnSegmentFile.Source( + dataFile("data-1", 3), + rawReader, + position -> position == 0)), + ROW_POSITION, vectorField, options, "definition", @@ -126,8 +132,7 @@ void testBuildSkipsNullAndSnapshotDeletedRows() throws Exception { "l2", "test-vector-ann", new byte[] {1, 2}, - 42, - position -> position == 0); + 42); } PkVectorSegmentMeta metadata = @@ -154,9 +159,11 @@ void testAnnSearchUsesRowPositionDeletionMask() throws Exception { IndexFileMeta segment; try (RawVectorSidecarReader rawReader = new RawVectorSidecarReader(fileIO, rawPath)) { segment = - annFile.buildSingleSource( - dataFile("data-1", 3), - rawReader, + annFile.build( + Collections.singletonList( + new PkVectorAnnSegmentFile.Source( + dataFile("data-1", 3), rawReader)), + ROW_POSITION, vectorField, options, "definition", @@ -164,8 +171,7 @@ void testAnnSearchUsesRowPositionDeletionMask() throws Exception { "l2", "test-vector-ann", new byte[] {1, 2}, - 42, - position -> false); + 42); } PkVectorSegmentMeta metadata = PkVectorSegmentMeta.deserialize(segment.globalIndexMeta().indexMeta()); @@ -217,10 +223,11 @@ void testBuildsAndSearchesMultiSourceAnnSegment() throws Exception { try (RawVectorSidecarReader raw1 = new RawVectorSidecarReader(fileIO, raw1Path); RawVectorSidecarReader raw2 = new RawVectorSidecarReader(fileIO, raw2Path)) { segment = - annFile.buildMultiSource( + annFile.build( Arrays.asList( new PkVectorAnnSegmentFile.Source(dataFile("data-1"), raw1), new PkVectorAnnSegmentFile.Source(dataFile("data-2"), raw2)), + FILE_POSITION, vectorField, options, "definition", From cd7b714ab804f46c54d5b058d60adb78f3e439a3 Mon Sep 17 00:00:00 2001 From: JingsongLi Date: Sat, 11 Jul 2026 18:37:49 +0800 Subject: [PATCH 10/19] [core] Minimize primary-key vector segment metadata --- .../pkvector/PkVectorAnnSegmentFile.java | 26 +-- .../pkvector/PkVectorAnnSegmentSearcher.java | 21 +-- .../pkvector/PkVectorRawSegmentFile.java | 36 +--- .../index/pkvector/PkVectorSegmentMeta.java | 164 +----------------- .../pkvector/PkVectorAnnSegmentFileTest.java | 45 +++-- .../pkvector/PkVectorSegmentMetaTest.java | 43 +---- 6 files changed, 43 insertions(+), 292 deletions(-) diff --git a/paimon-core/src/main/java/org/apache/paimon/index/pkvector/PkVectorAnnSegmentFile.java b/paimon-core/src/main/java/org/apache/paimon/index/pkvector/PkVectorAnnSegmentFile.java index e899fc07d812..110c221440a1 100644 --- a/paimon-core/src/main/java/org/apache/paimon/index/pkvector/PkVectorAnnSegmentFile.java +++ b/paimon-core/src/main/java/org/apache/paimon/index/pkvector/PkVectorAnnSegmentFile.java @@ -47,7 +47,6 @@ import java.util.Map; import java.util.function.LongPredicate; -import static org.apache.paimon.index.pkvector.PkVectorSegmentMeta.Role.ANN; import static org.apache.paimon.utils.Preconditions.checkArgument; /** Builds immutable ANN payloads whose index ids are source data-file row positions. */ @@ -65,11 +64,8 @@ public IndexFileMeta build( DataField vectorField, Options indexOptions, String indexDefinitionId, - String vectorTypeFingerprint, String metric, - String algorithm, - byte[] optionsHash, - long buildSnapshotId) + String algorithm) throws IOException { checkArgument(!sources.isEmpty(), "An ANN segment must reference source files."); long totalRowCount = 0; @@ -157,18 +153,7 @@ public IndexFileMeta build( byte[] payloadMetadata = result.meta() == null ? new byte[0] : result.meta(); PkVectorSegmentMeta metadata = new PkVectorSegmentMeta( - ANN, - indexDefinitionId, - vectorField.id(), - vectorTypeFingerprint, - normalizeMetric(metric), - algorithm, - sourceFiles, - ordinalLayout, - liveRowCount, - buildSnapshotId, - optionsHash, - payloadMetadata); + indexDefinitionId, sourceFiles, ordinalLayout, payloadMetadata); IndexFileMeta segment = new IndexFileMeta( PK_VECTOR_ANN, @@ -191,12 +176,7 @@ public IndexFileMeta build( } private static PkVectorSegmentMeta.SourceFile sourceMetadata(DataFileMeta sourceFile) { - return new PkVectorSegmentMeta.SourceFile( - sourceFile.fileName(), - sourceFile.schemaId(), - sourceFile.level(), - sourceFile.rowCount(), - sourceFile.fileSize()); + return new PkVectorSegmentMeta.SourceFile(sourceFile.fileName(), sourceFile.rowCount()); } private static String normalizeMetric(String metric) { diff --git a/paimon-core/src/main/java/org/apache/paimon/index/pkvector/PkVectorAnnSegmentSearcher.java b/paimon-core/src/main/java/org/apache/paimon/index/pkvector/PkVectorAnnSegmentSearcher.java index 2ba7daae3596..5ed2b2e54888 100644 --- a/paimon-core/src/main/java/org/apache/paimon/index/pkvector/PkVectorAnnSegmentSearcher.java +++ b/paimon-core/src/main/java/org/apache/paimon/index/pkvector/PkVectorAnnSegmentSearcher.java @@ -47,7 +47,6 @@ import static org.apache.paimon.index.pkvector.PkVectorSegmentMeta.OrdinalLayout.FILE_POSITION; import static org.apache.paimon.index.pkvector.PkVectorSegmentMeta.OrdinalLayout.ROW_POSITION; -import static org.apache.paimon.index.pkvector.PkVectorSegmentMeta.Role.ANN; import static org.apache.paimon.utils.Preconditions.checkArgument; /** Searches one ANN payload and maps its segment-local ids back to source row positions. */ @@ -62,6 +61,8 @@ public class PkVectorAnnSegmentSearcher { private final PkVectorAnnSegmentFile annSegmentFile; private final DataField vectorField; private final Options indexOptions; + private final String algorithm; + private final String metric; private final ExecutorService executor; public PkVectorAnnSegmentSearcher( @@ -69,11 +70,15 @@ public PkVectorAnnSegmentSearcher( PkVectorAnnSegmentFile annSegmentFile, DataField vectorField, Options indexOptions, + String algorithm, + String metric, ExecutorService executor) { this.fileIO = fileIO; this.annSegmentFile = annSegmentFile; this.vectorField = vectorField; this.indexOptions = indexOptions; + this.algorithm = algorithm; + this.metric = normalizeMetric(metric); this.executor = executor; } @@ -106,7 +111,6 @@ public List search( PkVectorAnnSegmentFile.PK_VECTOR_ANN.equals(segment.indexType()), "Vector segment %s is not an ANN payload.", segment.fileName()); - checkArgument(metadata.role() == ANN, "Vector segment %s is not ANN.", segment.fileName()); checkArgument( metadata.ordinalLayout() == ROW_POSITION || metadata.ordinalLayout() == FILE_POSITION, @@ -117,20 +121,11 @@ public List search( metadata.ordinalLayout() != ROW_POSITION || metadata.sourceFiles().size() == 1, "Row-position ANN segment %s must reference exactly one source file.", segment.fileName()); - checkArgument( - metadata.vectorFieldId() == vectorField.id(), - "ANN segment %s has vector field %s, but reader expects %s.", - segment.fileName(), - metadata.vectorFieldId(), - vectorField.id()); - - GlobalIndexer indexer = - GlobalIndexer.create(metadata.algorithm(), vectorField, indexOptions); + GlobalIndexer indexer = GlobalIndexer.create(algorithm, vectorField, indexOptions); checkArgument( indexer instanceof VectorGlobalIndexer, "Index algorithm %s does not implement VectorGlobalIndexer.", - metadata.algorithm()); - String metric = normalizeMetric(metadata.metric()); + algorithm); String readerMetric = normalizeMetric(((VectorGlobalIndexer) indexer).metric()); checkArgument( metric.equals(readerMetric), diff --git a/paimon-core/src/main/java/org/apache/paimon/index/pkvector/PkVectorRawSegmentFile.java b/paimon-core/src/main/java/org/apache/paimon/index/pkvector/PkVectorRawSegmentFile.java index a8d0b129d26d..e1dd7b1c9018 100644 --- a/paimon-core/src/main/java/org/apache/paimon/index/pkvector/PkVectorRawSegmentFile.java +++ b/paimon-core/src/main/java/org/apache/paimon/index/pkvector/PkVectorRawSegmentFile.java @@ -40,7 +40,6 @@ import java.util.function.Consumer; import static org.apache.paimon.index.pkvector.PkVectorSegmentMeta.OrdinalLayout.ROW_POSITION; -import static org.apache.paimon.index.pkvector.PkVectorSegmentMeta.Role.RAW_DELTA; import static org.apache.paimon.utils.Preconditions.checkArgument; import static org.apache.paimon.utils.Preconditions.checkState; @@ -61,10 +60,6 @@ public KeyValueVectorSidecarWriter newWriter( int dimension, String indexDefinitionId, int vectorFieldId, - String vectorTypeFingerprint, - String metric, - String algorithm, - byte[] optionsHash, BiConsumer segmentConsumer, Consumer segmentAbortConsumer) { Path path = pathFactory.newPath(); @@ -73,10 +68,6 @@ public KeyValueVectorSidecarWriter newWriter( new RawVectorSidecarWriter(fileIO, path, dimension), indexDefinitionId, vectorFieldId, - vectorTypeFingerprint, - metric, - algorithm, - optionsHash, segmentConsumer, segmentAbortConsumer); } catch (IOException e) { @@ -91,10 +82,6 @@ private class Writer implements KeyValueVectorSidecarWriter { private final RawVectorSidecarWriter rawWriter; private final String indexDefinitionId; private final int vectorFieldId; - private final String vectorTypeFingerprint; - private final String metric; - private final String algorithm; - private final byte[] optionsHash; private final BiConsumer segmentConsumer; private final Consumer segmentAbortConsumer; @@ -106,19 +93,11 @@ private Writer( RawVectorSidecarWriter rawWriter, String indexDefinitionId, int vectorFieldId, - String vectorTypeFingerprint, - String metric, - String algorithm, - byte[] optionsHash, BiConsumer segmentConsumer, Consumer segmentAbortConsumer) { this.rawWriter = rawWriter; this.indexDefinitionId = indexDefinitionId; this.vectorFieldId = vectorFieldId; - this.vectorTypeFingerprint = vectorTypeFingerprint; - this.metric = metric; - this.algorithm = algorithm; - this.optionsHash = optionsHash.clone(); this.segmentConsumer = segmentConsumer; this.segmentAbortConsumer = segmentAbortConsumer; } @@ -149,23 +128,12 @@ public void complete(DataFileMeta sourceFile) throws IOException { PkVectorSegmentMeta metadata = new PkVectorSegmentMeta( - RAW_DELTA, indexDefinitionId, - vectorFieldId, - vectorTypeFingerprint, - metric, - algorithm, Collections.singletonList( new PkVectorSegmentMeta.SourceFile( - sourceFile.fileName(), - sourceFile.schemaId(), - sourceFile.level(), - sourceFile.rowCount(), - sourceFile.fileSize())), + sourceFile.fileName(), sourceFile.rowCount())), ROW_POSITION, - rawWriter.liveVectorCount(), - 0, - optionsHash); + new byte[0]); Path path = rawWriter.path(); IndexFileMeta segment = new IndexFileMeta( diff --git a/paimon-core/src/main/java/org/apache/paimon/index/pkvector/PkVectorSegmentMeta.java b/paimon-core/src/main/java/org/apache/paimon/index/pkvector/PkVectorSegmentMeta.java index c9831f741537..8dc00d49a117 100644 --- a/paimon-core/src/main/java/org/apache/paimon/index/pkvector/PkVectorSegmentMeta.java +++ b/paimon-core/src/main/java/org/apache/paimon/index/pkvector/PkVectorSegmentMeta.java @@ -35,101 +35,28 @@ public class PkVectorSegmentMeta { private static final int VERSION = 1; - private final Role role; private final String indexDefinitionId; - private final int vectorFieldId; - private final String vectorTypeFingerprint; - private final String metric; - private final String algorithm; private final List sourceFiles; private final OrdinalLayout ordinalLayout; - private final long liveRowCountAtBuild; - private final long buildSnapshotId; - private final byte[] optionsHash; private final byte[] payloadMetadata; public PkVectorSegmentMeta( - Role role, String indexDefinitionId, - int vectorFieldId, - String vectorTypeFingerprint, - String metric, - String algorithm, List sourceFiles, OrdinalLayout ordinalLayout, - long liveRowCountAtBuild, - long buildSnapshotId, - byte[] optionsHash) { - this( - role, - indexDefinitionId, - vectorFieldId, - vectorTypeFingerprint, - metric, - algorithm, - sourceFiles, - ordinalLayout, - liveRowCountAtBuild, - buildSnapshotId, - optionsHash, - new byte[0]); - } - - public PkVectorSegmentMeta( - Role role, - String indexDefinitionId, - int vectorFieldId, - String vectorTypeFingerprint, - String metric, - String algorithm, - List sourceFiles, - OrdinalLayout ordinalLayout, - long liveRowCountAtBuild, - long buildSnapshotId, - byte[] optionsHash, byte[] payloadMetadata) { - this.role = Objects.requireNonNull(role); this.indexDefinitionId = Objects.requireNonNull(indexDefinitionId); - this.vectorFieldId = vectorFieldId; - this.vectorTypeFingerprint = Objects.requireNonNull(vectorTypeFingerprint); - this.metric = Objects.requireNonNull(metric); - this.algorithm = Objects.requireNonNull(algorithm); this.sourceFiles = Collections.unmodifiableList(new ArrayList<>(sourceFiles)); this.ordinalLayout = Objects.requireNonNull(ordinalLayout); - this.liveRowCountAtBuild = liveRowCountAtBuild; - this.buildSnapshotId = buildSnapshotId; - this.optionsHash = Arrays.copyOf(optionsHash, optionsHash.length); this.payloadMetadata = Arrays.copyOf(payloadMetadata, payloadMetadata.length); checkArgument(!this.sourceFiles.isEmpty(), "A vector segment must reference source files."); - checkArgument(liveRowCountAtBuild >= 0, "Live row count must not be negative."); - checkArgument(buildSnapshotId >= 0, "Build snapshot id must not be negative."); - } - - public Role role() { - return role; } public String indexDefinitionId() { return indexDefinitionId; } - public int vectorFieldId() { - return vectorFieldId; - } - - public String vectorTypeFingerprint() { - return vectorTypeFingerprint; - } - - public String metric() { - return metric; - } - - public String algorithm() { - return algorithm; - } - public List sourceFiles() { return sourceFiles; } @@ -138,18 +65,6 @@ public OrdinalLayout ordinalLayout() { return ordinalLayout; } - public long liveRowCountAtBuild() { - return liveRowCountAtBuild; - } - - public long buildSnapshotId() { - return buildSnapshotId; - } - - public byte[] optionsHash() { - return Arrays.copyOf(optionsHash, optionsHash.length); - } - public byte[] payloadMetadata() { return Arrays.copyOf(payloadMetadata, payloadMetadata.length); } @@ -159,25 +74,13 @@ public byte[] serialize() { try { DataOutputSerializer output = new DataOutputSerializer(128); output.writeInt(VERSION); - output.writeByte(role.ordinal()); output.writeUTF(indexDefinitionId); - output.writeInt(vectorFieldId); - output.writeUTF(vectorTypeFingerprint); - output.writeUTF(metric); - output.writeUTF(algorithm); output.writeInt(sourceFiles.size()); for (SourceFile sourceFile : sourceFiles) { output.writeUTF(sourceFile.fileName); - output.writeLong(sourceFile.schemaId); - output.writeInt(sourceFile.level); output.writeLong(sourceFile.rowCount); - output.writeLong(sourceFile.fileSize); } output.writeByte(ordinalLayout.ordinal()); - output.writeLong(liveRowCountAtBuild); - output.writeLong(buildSnapshotId); - output.writeInt(optionsHash.length); - output.write(optionsHash); output.writeInt(payloadMetadata.length); output.write(payloadMetadata); return output.getCopyOfBuffer(); @@ -196,32 +99,15 @@ public static PkVectorSegmentMeta deserialize(byte[] bytes) { version == VERSION, "Unsupported primary-key vector segment version: %s.", version); - Role role = enumValue(Role.values(), input.readByte(), "role"); String indexDefinitionId = input.readUTF(); - int vectorFieldId = input.readInt(); - String vectorTypeFingerprint = input.readUTF(); - String metric = input.readUTF(); - String algorithm = input.readUTF(); int sourceFileCount = input.readInt(); checkArgument(sourceFileCount > 0, "A vector segment must reference source files."); List sourceFiles = new ArrayList<>(sourceFileCount); for (int i = 0; i < sourceFileCount; i++) { - sourceFiles.add( - new SourceFile( - input.readUTF(), - input.readLong(), - input.readInt(), - input.readLong(), - input.readLong())); + sourceFiles.add(new SourceFile(input.readUTF(), input.readLong())); } OrdinalLayout ordinalLayout = enumValue(OrdinalLayout.values(), input.readByte(), "ordinal layout"); - long liveRowCountAtBuild = input.readLong(); - long buildSnapshotId = input.readLong(); - int optionsHashLength = input.readInt(); - checkArgument(optionsHashLength >= 0, "Options hash length must not be negative."); - byte[] optionsHash = new byte[optionsHashLength]; - input.readFully(optionsHash); int payloadMetadataLength = input.readInt(); checkArgument( payloadMetadataLength >= 0, "Payload metadata length must not be negative."); @@ -231,18 +117,7 @@ public static PkVectorSegmentMeta deserialize(byte[] bytes) { input.available() == 0, "Unexpected trailing bytes in vector segment metadata."); return new PkVectorSegmentMeta( - role, - indexDefinitionId, - vectorFieldId, - vectorTypeFingerprint, - metric, - algorithm, - sourceFiles, - ordinalLayout, - liveRowCountAtBuild, - buildSnapshotId, - optionsHash, - payloadMetadata); + indexDefinitionId, sourceFiles, ordinalLayout, payloadMetadata); } catch (IOException e) { throw new IllegalArgumentException( "Failed to deserialize primary-key vector segment metadata.", e); @@ -259,12 +134,6 @@ private static T enumValue(T[] values, byte ordinal, String field) { return values[index]; } - /** Role of an immutable vector payload. */ - public enum Role { - RAW_DELTA, - ANN - } - /** Mapping from a segment-local ordinal to a physical data-file position. */ public enum OrdinalLayout { ROW_POSITION, @@ -275,41 +144,22 @@ public enum OrdinalLayout { public static class SourceFile { private final String fileName; - private final long schemaId; - private final int level; private final long rowCount; - private final long fileSize; - public SourceFile(String fileName, long schemaId, int level, long rowCount, long fileSize) { + public SourceFile(String fileName, long rowCount) { this.fileName = Objects.requireNonNull(fileName); - this.schemaId = schemaId; - this.level = level; this.rowCount = rowCount; - this.fileSize = fileSize; checkArgument(rowCount >= 0, "Source file row count must not be negative."); - checkArgument(fileSize >= 0, "Source file size must not be negative."); } public String fileName() { return fileName; } - public long schemaId() { - return schemaId; - } - - public int level() { - return level; - } - public long rowCount() { return rowCount; } - public long fileSize() { - return fileSize; - } - @Override public boolean equals(Object o) { if (this == o) { @@ -319,16 +169,12 @@ public boolean equals(Object o) { return false; } SourceFile that = (SourceFile) o; - return schemaId == that.schemaId - && level == that.level - && rowCount == that.rowCount - && fileSize == that.fileSize - && Objects.equals(fileName, that.fileName); + return rowCount == that.rowCount && Objects.equals(fileName, that.fileName); } @Override public int hashCode() { - return Objects.hash(fileName, schemaId, level, rowCount, fileSize); + return Objects.hash(fileName, rowCount); } } } diff --git a/paimon-core/src/test/java/org/apache/paimon/index/pkvector/PkVectorAnnSegmentFileTest.java b/paimon-core/src/test/java/org/apache/paimon/index/pkvector/PkVectorAnnSegmentFileTest.java index 64c1e93b0f11..dbb49d1e2f88 100644 --- a/paimon-core/src/test/java/org/apache/paimon/index/pkvector/PkVectorAnnSegmentFileTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/index/pkvector/PkVectorAnnSegmentFileTest.java @@ -44,7 +44,6 @@ import static org.apache.paimon.index.pkvector.PkVectorSegmentMeta.OrdinalLayout.FILE_POSITION; import static org.apache.paimon.index.pkvector.PkVectorSegmentMeta.OrdinalLayout.ROW_POSITION; -import static org.apache.paimon.index.pkvector.PkVectorSegmentMeta.Role.ANN; import static org.assertj.core.api.Assertions.assertThat; /** Tests ANN payload construction through the vector GlobalIndexer SPI. */ @@ -78,11 +77,8 @@ void testBuildsSingleSourceAnnSegmentWithRowPositionOrdinals() throws Exception vectorField, options, "definition", - "ARRAY", "l2", - "test-vector-ann", - new byte[] {1, 2}, - 42); + "test-vector-ann"); } assertThat(segment.indexType()).isEqualTo(PkVectorAnnSegmentFile.PK_VECTOR_ANN); @@ -90,13 +86,10 @@ void testBuildsSingleSourceAnnSegmentWithRowPositionOrdinals() throws Exception assertThat(fileIO.exists(pathFactory.toPath(segment))).isTrue(); PkVectorSegmentMeta metadata = PkVectorSegmentMeta.deserialize(segment.globalIndexMeta().indexMeta()); - assertThat(metadata.role()).isEqualTo(ANN); assertThat(metadata.ordinalLayout()).isEqualTo(ROW_POSITION); assertThat(metadata.sourceFiles()).hasSize(1); assertThat(metadata.sourceFiles().get(0).fileName()).isEqualTo("data-1"); - assertThat(metadata.liveRowCountAtBuild()).isEqualTo(2); - assertThat(metadata.buildSnapshotId()).isEqualTo(42); - assertThat(metadata.optionsHash()).containsExactly(1, 2); + assertThat(segment.globalIndexMeta().indexFieldId()).isEqualTo(7); } @Test @@ -128,16 +121,12 @@ void testBuildSkipsNullAndSnapshotDeletedRows() throws Exception { vectorField, options, "definition", - "ARRAY", "l2", - "test-vector-ann", - new byte[] {1, 2}, - 42); + "test-vector-ann"); } PkVectorSegmentMeta metadata = PkVectorSegmentMeta.deserialize(segment.globalIndexMeta().indexMeta()); - assertThat(metadata.liveRowCountAtBuild()).isEqualTo(1); assertThat(segment.rowCount()).isEqualTo(1); } @@ -167,11 +156,8 @@ void testAnnSearchUsesRowPositionDeletionMask() throws Exception { vectorField, options, "definition", - "ARRAY", "l2", - "test-vector-ann", - new byte[] {1, 2}, - 42); + "test-vector-ann"); } PkVectorSegmentMeta metadata = PkVectorSegmentMeta.deserialize(segment.globalIndexMeta().indexMeta()); @@ -181,7 +167,14 @@ void testAnnSearchUsesRowPositionDeletionMask() throws Exception { List candidates; try { candidates = - new PkVectorAnnSegmentSearcher(fileIO, annFile, vectorField, options, executor) + new PkVectorAnnSegmentSearcher( + fileIO, + annFile, + vectorField, + options, + "test-vector-ann", + "l2", + executor) .search( segment, metadata, @@ -231,11 +224,8 @@ void testBuildsAndSearchesMultiSourceAnnSegment() throws Exception { vectorField, options, "definition", - "ARRAY", "l2", - "test-vector-ann", - new byte[] {1, 2}, - 42); + "test-vector-ann"); } PkVectorSegmentMeta metadata = @@ -255,7 +245,14 @@ void testBuildsAndSearchesMultiSourceAnnSegment() throws Exception { List candidates; try { candidates = - new PkVectorAnnSegmentSearcher(fileIO, annFile, vectorField, options, executor) + new PkVectorAnnSegmentSearcher( + fileIO, + annFile, + vectorField, + options, + "test-vector-ann", + "l2", + executor) .search( segment, metadata, diff --git a/paimon-core/src/test/java/org/apache/paimon/index/pkvector/PkVectorSegmentMetaTest.java b/paimon-core/src/test/java/org/apache/paimon/index/pkvector/PkVectorSegmentMetaTest.java index afb0f068ed2e..ece36b55ebb6 100644 --- a/paimon-core/src/test/java/org/apache/paimon/index/pkvector/PkVectorSegmentMetaTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/index/pkvector/PkVectorSegmentMetaTest.java @@ -25,7 +25,6 @@ import java.util.Arrays; import static org.apache.paimon.index.pkvector.PkVectorSegmentMeta.OrdinalLayout.FILE_POSITION; -import static org.apache.paimon.index.pkvector.PkVectorSegmentMeta.Role.ANN; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; @@ -36,34 +35,18 @@ class PkVectorSegmentMetaTest { void testRoundTrip() { PkVectorSegmentMeta metadata = new PkVectorSegmentMeta( - ANN, "1d4502f1-9cf0-4d86-8d8d-5cc9ac05e108", - 7, - "VECTOR(1024)", - "l2", - "ivf-pq", Arrays.asList( - new PkVectorSegmentMeta.SourceFile("data-1", 3, 0, 100, 1024), - new PkVectorSegmentMeta.SourceFile("data-2", 3, 0, 50, 512)), + new PkVectorSegmentMeta.SourceFile("data-1", 100), + new PkVectorSegmentMeta.SourceFile("data-2", 50)), FILE_POSITION, - 120, - 42, - new byte[] {1, 2, 3}, new byte[] {4, 5, 6}); PkVectorSegmentMeta restored = PkVectorSegmentMeta.deserialize(metadata.serialize()); - assertThat(restored.role()).isEqualTo(ANN); assertThat(restored.indexDefinitionId()).isEqualTo(metadata.indexDefinitionId()); - assertThat(restored.vectorFieldId()).isEqualTo(7); - assertThat(restored.vectorTypeFingerprint()).isEqualTo("VECTOR(1024)"); - assertThat(restored.metric()).isEqualTo("l2"); - assertThat(restored.algorithm()).isEqualTo("ivf-pq"); assertThat(restored.sourceFiles()).isEqualTo(metadata.sourceFiles()); assertThat(restored.ordinalLayout()).isEqualTo(FILE_POSITION); - assertThat(restored.liveRowCountAtBuild()).isEqualTo(120); - assertThat(restored.buildSnapshotId()).isEqualTo(42); - assertThat(restored.optionsHash()).containsExactly(1, 2, 3); assertThat(restored.payloadMetadata()).containsExactly(4, 5, 6); } @@ -71,16 +54,9 @@ void testRoundTrip() { void testRejectTrailingBytes() { PkVectorSegmentMeta metadata = new PkVectorSegmentMeta( - ANN, "index", - 1, - "VECTOR(2)", - "l2", - "ivf-pq", - Arrays.asList(new PkVectorSegmentMeta.SourceFile("data", 1, 0, 1, 8)), + Arrays.asList(new PkVectorSegmentMeta.SourceFile("data", 1)), FILE_POSITION, - 1, - 1, new byte[0]); byte[] bytes = Arrays.copyOf(metadata.serialize(), metadata.serialize().length + 1); @@ -89,26 +65,15 @@ void testRejectTrailingBytes() { } @Test - void testRejectsPreReleaseLayoutWithoutPayloadMetadata() throws Exception { + void testRejectsTruncatedPayloadMetadata() throws Exception { DataOutputSerializer output = new DataOutputSerializer(128); output.writeInt(1); - output.writeByte(ANN.ordinal()); output.writeUTF("index"); - output.writeInt(7); - output.writeUTF("VECTOR(2)"); - output.writeUTF("l2"); - output.writeUTF("ivf-pq"); output.writeInt(1); output.writeUTF("data-1"); - output.writeLong(3); - output.writeInt(0); output.writeLong(10); - output.writeLong(100); output.writeByte(FILE_POSITION.ordinal()); - output.writeLong(9); - output.writeLong(42); output.writeInt(1); - output.writeByte(1); assertThatThrownBy(() -> PkVectorSegmentMeta.deserialize(output.getCopyOfBuffer())) .hasMessageContaining("Failed to deserialize primary-key vector segment metadata"); From 297e46e463c0e887c0c76782fa1d7547b4a70e56 Mon Sep 17 00:00:00 2001 From: JingsongLi Date: Sat, 11 Jul 2026 18:49:53 +0800 Subject: [PATCH 11/19] [core] Split RAW and ANN vector segment metadata --- .../pkvector/PkVectorAnnSegmentFile.java | 20 ++--- ...tMeta.java => PkVectorAnnSegmentMeta.java} | 86 +++++-------------- .../pkvector/PkVectorAnnSegmentSearcher.java | 22 +++-- .../pkvector/PkVectorRawSegmentFile.java | 12 +-- .../pkvector/PkVectorRawSegmentMeta.java | 81 +++++++++++++++++ .../index/pkvector/PkVectorSourceFile.java | 61 +++++++++++++ .../pkvector/PkVectorAnnSegmentFileTest.java | 24 +++--- .../pkvector/PkVectorAnnSegmentMetaTest.java | 67 +++++++++++++++ .../pkvector/PkVectorRawSegmentMetaTest.java | 52 +++++++++++ .../pkvector/PkVectorSegmentMetaTest.java | 81 ----------------- 10 files changed, 317 insertions(+), 189 deletions(-) rename paimon-core/src/main/java/org/apache/paimon/index/pkvector/{PkVectorSegmentMeta.java => PkVectorAnnSegmentMeta.java} (58%) create mode 100644 paimon-core/src/main/java/org/apache/paimon/index/pkvector/PkVectorRawSegmentMeta.java create mode 100644 paimon-core/src/main/java/org/apache/paimon/index/pkvector/PkVectorSourceFile.java create mode 100644 paimon-core/src/test/java/org/apache/paimon/index/pkvector/PkVectorAnnSegmentMetaTest.java create mode 100644 paimon-core/src/test/java/org/apache/paimon/index/pkvector/PkVectorRawSegmentMetaTest.java delete mode 100644 paimon-core/src/test/java/org/apache/paimon/index/pkvector/PkVectorSegmentMetaTest.java diff --git a/paimon-core/src/main/java/org/apache/paimon/index/pkvector/PkVectorAnnSegmentFile.java b/paimon-core/src/main/java/org/apache/paimon/index/pkvector/PkVectorAnnSegmentFile.java index 110c221440a1..c81625a34c5d 100644 --- a/paimon-core/src/main/java/org/apache/paimon/index/pkvector/PkVectorAnnSegmentFile.java +++ b/paimon-core/src/main/java/org/apache/paimon/index/pkvector/PkVectorAnnSegmentFile.java @@ -60,7 +60,7 @@ public PkVectorAnnSegmentFile(FileIO fileIO, IndexPathFactory pathFactory) { public IndexFileMeta build( List sources, - PkVectorSegmentMeta.OrdinalLayout ordinalLayout, + PkVectorAnnSegmentMeta.OrdinalLayout ordinalLayout, DataField vectorField, Options indexOptions, String indexDefinitionId, @@ -69,7 +69,7 @@ public IndexFileMeta build( throws IOException { checkArgument(!sources.isEmpty(), "An ANN segment must reference source files."); long totalRowCount = 0; - List sourceFiles = new ArrayList<>(sources.size()); + List sourceFiles = new ArrayList<>(sources.size()); for (Source source : sources) { totalRowCount = Math.addExact(totalRowCount, source.sourceFile.rowCount()); sourceFiles.add(source.sourceFile); @@ -151,8 +151,8 @@ public IndexFileMeta build( ResultEntry result = results.get(0); Path payloadPath = fileWriter.path(result.fileName()); byte[] payloadMetadata = result.meta() == null ? new byte[0] : result.meta(); - PkVectorSegmentMeta metadata = - new PkVectorSegmentMeta( + PkVectorAnnSegmentMeta metadata = + new PkVectorAnnSegmentMeta( indexDefinitionId, sourceFiles, ordinalLayout, payloadMetadata); IndexFileMeta segment = new IndexFileMeta( @@ -175,8 +175,8 @@ public IndexFileMeta build( } } - private static PkVectorSegmentMeta.SourceFile sourceMetadata(DataFileMeta sourceFile) { - return new PkVectorSegmentMeta.SourceFile(sourceFile.fileName(), sourceFile.rowCount()); + private static PkVectorSourceFile sourceMetadata(DataFileMeta sourceFile) { + return new PkVectorSourceFile(sourceFile.fileName(), sourceFile.rowCount()); } private static String normalizeMetric(String metric) { @@ -215,7 +215,7 @@ private void deleteCreatedFiles() { /** One raw vector source used while building an ANN segment. */ public static class Source { - private final PkVectorSegmentMeta.SourceFile sourceFile; + private final PkVectorSourceFile sourceFile; @Nullable private final RawVectorSidecarReader rawVectors; @Nullable private final ReaderFactory readerFactory; private final LongPredicate excludedPosition; @@ -232,7 +232,7 @@ public Source( } Source( - PkVectorSegmentMeta.SourceFile sourceFile, + PkVectorSourceFile sourceFile, RawVectorSidecarReader rawVectors, LongPredicate excludedPosition) { this.sourceFile = sourceFile; @@ -242,7 +242,7 @@ public Source( } private Source( - PkVectorSegmentMeta.SourceFile sourceFile, + PkVectorSourceFile sourceFile, ReaderFactory readerFactory, LongPredicate excludedPosition) { this.sourceFile = sourceFile; @@ -251,7 +251,7 @@ private Source( this.excludedPosition = excludedPosition; } - static Source lazy(PkVectorSegmentMeta.SourceFile sourceFile, ReaderFactory readerFactory) { + static Source lazy(PkVectorSourceFile sourceFile, ReaderFactory readerFactory) { return new Source(sourceFile, readerFactory, position -> false); } diff --git a/paimon-core/src/main/java/org/apache/paimon/index/pkvector/PkVectorSegmentMeta.java b/paimon-core/src/main/java/org/apache/paimon/index/pkvector/PkVectorAnnSegmentMeta.java similarity index 58% rename from paimon-core/src/main/java/org/apache/paimon/index/pkvector/PkVectorSegmentMeta.java rename to paimon-core/src/main/java/org/apache/paimon/index/pkvector/PkVectorAnnSegmentMeta.java index 8dc00d49a117..df8eaa3e83d2 100644 --- a/paimon-core/src/main/java/org/apache/paimon/index/pkvector/PkVectorSegmentMeta.java +++ b/paimon-core/src/main/java/org/apache/paimon/index/pkvector/PkVectorAnnSegmentMeta.java @@ -30,34 +30,33 @@ import static org.apache.paimon.utils.Preconditions.checkArgument; -/** Versioned metadata for an immutable primary-key vector segment. */ -public class PkVectorSegmentMeta { +/** Versioned metadata for an immutable ANN primary-key vector segment. */ +public final class PkVectorAnnSegmentMeta { private static final int VERSION = 1; private final String indexDefinitionId; - private final List sourceFiles; + private final List sourceFiles; private final OrdinalLayout ordinalLayout; private final byte[] payloadMetadata; - public PkVectorSegmentMeta( + public PkVectorAnnSegmentMeta( String indexDefinitionId, - List sourceFiles, + List sourceFiles, OrdinalLayout ordinalLayout, byte[] payloadMetadata) { this.indexDefinitionId = Objects.requireNonNull(indexDefinitionId); this.sourceFiles = Collections.unmodifiableList(new ArrayList<>(sourceFiles)); this.ordinalLayout = Objects.requireNonNull(ordinalLayout); this.payloadMetadata = Arrays.copyOf(payloadMetadata, payloadMetadata.length); - - checkArgument(!this.sourceFiles.isEmpty(), "A vector segment must reference source files."); + checkArgument(!this.sourceFiles.isEmpty(), "An ANN segment must reference source files."); } public String indexDefinitionId() { return indexDefinitionId; } - public List sourceFiles() { + public List sourceFiles() { return sourceFiles; } @@ -69,42 +68,37 @@ public byte[] payloadMetadata() { return Arrays.copyOf(payloadMetadata, payloadMetadata.length); } - /** Serializes this metadata for {@link org.apache.paimon.index.GlobalIndexMeta#indexMeta()}. */ public byte[] serialize() { try { DataOutputSerializer output = new DataOutputSerializer(128); output.writeInt(VERSION); output.writeUTF(indexDefinitionId); output.writeInt(sourceFiles.size()); - for (SourceFile sourceFile : sourceFiles) { - output.writeUTF(sourceFile.fileName); - output.writeLong(sourceFile.rowCount); + for (PkVectorSourceFile sourceFile : sourceFiles) { + output.writeUTF(sourceFile.fileName()); + output.writeLong(sourceFile.rowCount()); } output.writeByte(ordinalLayout.ordinal()); output.writeInt(payloadMetadata.length); output.write(payloadMetadata); return output.getCopyOfBuffer(); } catch (IOException e) { - throw new RuntimeException( - "Failed to serialize primary-key vector segment metadata.", e); + throw new RuntimeException("Failed to serialize ANN vector segment metadata.", e); } } - /** Deserializes primary-key vector metadata stored in {@code GlobalIndexMeta.indexMeta}. */ - public static PkVectorSegmentMeta deserialize(byte[] bytes) { + public static PkVectorAnnSegmentMeta deserialize(byte[] bytes) { try { DataInputDeserializer input = new DataInputDeserializer(bytes); int version = input.readInt(); checkArgument( - version == VERSION, - "Unsupported primary-key vector segment version: %s.", - version); + version == VERSION, "Unsupported ANN vector segment version: %s.", version); String indexDefinitionId = input.readUTF(); int sourceFileCount = input.readInt(); - checkArgument(sourceFileCount > 0, "A vector segment must reference source files."); - List sourceFiles = new ArrayList<>(sourceFileCount); + checkArgument(sourceFileCount > 0, "An ANN segment must reference source files."); + List sourceFiles = new ArrayList<>(sourceFileCount); for (int i = 0; i < sourceFileCount; i++) { - sourceFiles.add(new SourceFile(input.readUTF(), input.readLong())); + sourceFiles.add(new PkVectorSourceFile(input.readUTF(), input.readLong())); } OrdinalLayout ordinalLayout = enumValue(OrdinalLayout.values(), input.readByte(), "ordinal layout"); @@ -115,12 +109,12 @@ public static PkVectorSegmentMeta deserialize(byte[] bytes) { input.readFully(payloadMetadata); checkArgument( input.available() == 0, - "Unexpected trailing bytes in vector segment metadata."); - return new PkVectorSegmentMeta( + "Unexpected trailing bytes in ANN vector segment metadata."); + return new PkVectorAnnSegmentMeta( indexDefinitionId, sourceFiles, ordinalLayout, payloadMetadata); } catch (IOException e) { throw new IllegalArgumentException( - "Failed to deserialize primary-key vector segment metadata.", e); + "Failed to deserialize ANN vector segment metadata.", e); } } @@ -128,53 +122,15 @@ private static T enumValue(T[] values, byte ordinal, String field) { int index = ordinal; checkArgument( index >= 0 && index < values.length, - "Unknown vector segment %s: %s.", + "Unknown ANN vector segment %s: %s.", field, ordinal); return values[index]; } - /** Mapping from a segment-local ordinal to a physical data-file position. */ + /** Mapping from an ANN-local ordinal to a physical data-file position. */ public enum OrdinalLayout { ROW_POSITION, FILE_POSITION } - - /** Immutable source data-file identity captured when a vector segment is built. */ - public static class SourceFile { - - private final String fileName; - private final long rowCount; - - public SourceFile(String fileName, long rowCount) { - this.fileName = Objects.requireNonNull(fileName); - this.rowCount = rowCount; - checkArgument(rowCount >= 0, "Source file row count must not be negative."); - } - - public String fileName() { - return fileName; - } - - public long rowCount() { - return rowCount; - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - SourceFile that = (SourceFile) o; - return rowCount == that.rowCount && Objects.equals(fileName, that.fileName); - } - - @Override - public int hashCode() { - return Objects.hash(fileName, rowCount); - } - } } diff --git a/paimon-core/src/main/java/org/apache/paimon/index/pkvector/PkVectorAnnSegmentSearcher.java b/paimon-core/src/main/java/org/apache/paimon/index/pkvector/PkVectorAnnSegmentSearcher.java index 5ed2b2e54888..9196c21de3d2 100644 --- a/paimon-core/src/main/java/org/apache/paimon/index/pkvector/PkVectorAnnSegmentSearcher.java +++ b/paimon-core/src/main/java/org/apache/paimon/index/pkvector/PkVectorAnnSegmentSearcher.java @@ -45,8 +45,8 @@ import java.util.Optional; import java.util.concurrent.ExecutorService; -import static org.apache.paimon.index.pkvector.PkVectorSegmentMeta.OrdinalLayout.FILE_POSITION; -import static org.apache.paimon.index.pkvector.PkVectorSegmentMeta.OrdinalLayout.ROW_POSITION; +import static org.apache.paimon.index.pkvector.PkVectorAnnSegmentMeta.OrdinalLayout.FILE_POSITION; +import static org.apache.paimon.index.pkvector.PkVectorAnnSegmentMeta.OrdinalLayout.ROW_POSITION; import static org.apache.paimon.utils.Preconditions.checkArgument; /** Searches one ANN payload and maps its segment-local ids back to source row positions. */ @@ -84,7 +84,7 @@ public PkVectorAnnSegmentSearcher( public List search( IndexFileMeta segment, - PkVectorSegmentMeta metadata, + PkVectorAnnSegmentMeta metadata, float[] query, int limit, @Nullable DeletionVector deletionVector, @@ -101,7 +101,7 @@ public List search( public List search( IndexFileMeta segment, - PkVectorSegmentMeta metadata, + PkVectorAnnSegmentMeta metadata, float[] query, int limit, Map deletionVectors, @@ -188,15 +188,14 @@ public List search( @Nullable private static RoaringNavigableMap64 liveRowPositions( - List sourceFiles, - Map deletionVectors) { + List sourceFiles, Map deletionVectors) { if (deletionVectors.isEmpty()) { return null; } RoaringNavigableMap64 live = new RoaringNavigableMap64(); RoaringNavigableMap64 deleted = new RoaringNavigableMap64(); long fileOffset = 0; - for (PkVectorSegmentMeta.SourceFile sourceFile : sourceFiles) { + for (PkVectorSourceFile sourceFile : sourceFiles) { if (sourceFile.rowCount() > 0) { live.addRange(new Range(fileOffset, fileOffset + sourceFile.rowCount() - 1)); } @@ -211,18 +210,17 @@ private static RoaringNavigableMap64 liveRowPositions( return live; } - private static long totalRowCount(List sourceFiles) { + private static long totalRowCount(List sourceFiles) { long total = 0; - for (PkVectorSegmentMeta.SourceFile sourceFile : sourceFiles) { + for (PkVectorSourceFile sourceFile : sourceFiles) { total = Math.addExact(total, sourceFile.rowCount()); } return total; } - private static FilePosition filePosition( - List sourceFiles, long ordinal) { + private static FilePosition filePosition(List sourceFiles, long ordinal) { long fileOffset = 0; - for (PkVectorSegmentMeta.SourceFile sourceFile : sourceFiles) { + for (PkVectorSourceFile sourceFile : sourceFiles) { long nextOffset = fileOffset + sourceFile.rowCount(); if (ordinal < nextOffset) { return new FilePosition(sourceFile.fileName(), ordinal - fileOffset); diff --git a/paimon-core/src/main/java/org/apache/paimon/index/pkvector/PkVectorRawSegmentFile.java b/paimon-core/src/main/java/org/apache/paimon/index/pkvector/PkVectorRawSegmentFile.java index e1dd7b1c9018..d0a3a6ffe2f4 100644 --- a/paimon-core/src/main/java/org/apache/paimon/index/pkvector/PkVectorRawSegmentFile.java +++ b/paimon-core/src/main/java/org/apache/paimon/index/pkvector/PkVectorRawSegmentFile.java @@ -34,12 +34,10 @@ import java.io.IOException; import java.io.UncheckedIOException; -import java.util.Collections; import java.util.Optional; import java.util.function.BiConsumer; import java.util.function.Consumer; -import static org.apache.paimon.index.pkvector.PkVectorSegmentMeta.OrdinalLayout.ROW_POSITION; import static org.apache.paimon.utils.Preconditions.checkArgument; import static org.apache.paimon.utils.Preconditions.checkState; @@ -126,14 +124,10 @@ public void complete(DataFileMeta sourceFile) throws IOException { sourceFile.fileName(), sourceFile.rowCount()); - PkVectorSegmentMeta metadata = - new PkVectorSegmentMeta( + PkVectorRawSegmentMeta metadata = + new PkVectorRawSegmentMeta( indexDefinitionId, - Collections.singletonList( - new PkVectorSegmentMeta.SourceFile( - sourceFile.fileName(), sourceFile.rowCount())), - ROW_POSITION, - new byte[0]); + new PkVectorSourceFile(sourceFile.fileName(), sourceFile.rowCount())); Path path = rawWriter.path(); IndexFileMeta segment = new IndexFileMeta( diff --git a/paimon-core/src/main/java/org/apache/paimon/index/pkvector/PkVectorRawSegmentMeta.java b/paimon-core/src/main/java/org/apache/paimon/index/pkvector/PkVectorRawSegmentMeta.java new file mode 100644 index 000000000000..8b4e63ee9a93 --- /dev/null +++ b/paimon-core/src/main/java/org/apache/paimon/index/pkvector/PkVectorRawSegmentMeta.java @@ -0,0 +1,81 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.index.pkvector; + +import org.apache.paimon.io.DataInputDeserializer; +import org.apache.paimon.io.DataOutputSerializer; + +import java.io.IOException; +import java.util.Objects; + +import static org.apache.paimon.utils.Preconditions.checkArgument; + +/** Versioned metadata for an immutable RAW primary-key vector segment. */ +public final class PkVectorRawSegmentMeta { + + private static final int VERSION = 1; + + private final String indexDefinitionId; + private final PkVectorSourceFile sourceFile; + + public PkVectorRawSegmentMeta(String indexDefinitionId, PkVectorSourceFile sourceFile) { + this.indexDefinitionId = Objects.requireNonNull(indexDefinitionId); + this.sourceFile = Objects.requireNonNull(sourceFile); + } + + public String indexDefinitionId() { + return indexDefinitionId; + } + + public PkVectorSourceFile sourceFile() { + return sourceFile; + } + + public byte[] serialize() { + try { + DataOutputSerializer output = new DataOutputSerializer(64); + output.writeInt(VERSION); + output.writeUTF(indexDefinitionId); + output.writeUTF(sourceFile.fileName()); + output.writeLong(sourceFile.rowCount()); + return output.getCopyOfBuffer(); + } catch (IOException e) { + throw new RuntimeException("Failed to serialize RAW vector segment metadata.", e); + } + } + + public static PkVectorRawSegmentMeta deserialize(byte[] bytes) { + try { + DataInputDeserializer input = new DataInputDeserializer(bytes); + int version = input.readInt(); + checkArgument( + version == VERSION, "Unsupported RAW vector segment version: %s.", version); + String indexDefinitionId = input.readUTF(); + PkVectorSourceFile sourceFile = + new PkVectorSourceFile(input.readUTF(), input.readLong()); + checkArgument( + input.available() == 0, + "Unexpected trailing bytes in RAW vector segment metadata."); + return new PkVectorRawSegmentMeta(indexDefinitionId, sourceFile); + } catch (IOException e) { + throw new IllegalArgumentException( + "Failed to deserialize RAW vector segment metadata.", e); + } + } +} diff --git a/paimon-core/src/main/java/org/apache/paimon/index/pkvector/PkVectorSourceFile.java b/paimon-core/src/main/java/org/apache/paimon/index/pkvector/PkVectorSourceFile.java new file mode 100644 index 000000000000..87429d5694d5 --- /dev/null +++ b/paimon-core/src/main/java/org/apache/paimon/index/pkvector/PkVectorSourceFile.java @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.index.pkvector; + +import java.util.Objects; + +import static org.apache.paimon.utils.Preconditions.checkArgument; + +/** Immutable source data-file identity captured when a vector segment is built. */ +public final class PkVectorSourceFile { + + private final String fileName; + private final long rowCount; + + public PkVectorSourceFile(String fileName, long rowCount) { + this.fileName = Objects.requireNonNull(fileName); + this.rowCount = rowCount; + checkArgument(rowCount >= 0, "Source file row count must not be negative."); + } + + public String fileName() { + return fileName; + } + + public long rowCount() { + return rowCount; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + PkVectorSourceFile that = (PkVectorSourceFile) o; + return rowCount == that.rowCount && Objects.equals(fileName, that.fileName); + } + + @Override + public int hashCode() { + return Objects.hash(fileName, rowCount); + } +} diff --git a/paimon-core/src/test/java/org/apache/paimon/index/pkvector/PkVectorAnnSegmentFileTest.java b/paimon-core/src/test/java/org/apache/paimon/index/pkvector/PkVectorAnnSegmentFileTest.java index dbb49d1e2f88..03138e81eea2 100644 --- a/paimon-core/src/test/java/org/apache/paimon/index/pkvector/PkVectorAnnSegmentFileTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/index/pkvector/PkVectorAnnSegmentFileTest.java @@ -42,8 +42,8 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; -import static org.apache.paimon.index.pkvector.PkVectorSegmentMeta.OrdinalLayout.FILE_POSITION; -import static org.apache.paimon.index.pkvector.PkVectorSegmentMeta.OrdinalLayout.ROW_POSITION; +import static org.apache.paimon.index.pkvector.PkVectorAnnSegmentMeta.OrdinalLayout.FILE_POSITION; +import static org.apache.paimon.index.pkvector.PkVectorAnnSegmentMeta.OrdinalLayout.ROW_POSITION; import static org.assertj.core.api.Assertions.assertThat; /** Tests ANN payload construction through the vector GlobalIndexer SPI. */ @@ -84,8 +84,8 @@ void testBuildsSingleSourceAnnSegmentWithRowPositionOrdinals() throws Exception assertThat(segment.indexType()).isEqualTo(PkVectorAnnSegmentFile.PK_VECTOR_ANN); assertThat(segment.rowCount()).isEqualTo(2); assertThat(fileIO.exists(pathFactory.toPath(segment))).isTrue(); - PkVectorSegmentMeta metadata = - PkVectorSegmentMeta.deserialize(segment.globalIndexMeta().indexMeta()); + PkVectorAnnSegmentMeta metadata = + PkVectorAnnSegmentMeta.deserialize(segment.globalIndexMeta().indexMeta()); assertThat(metadata.ordinalLayout()).isEqualTo(ROW_POSITION); assertThat(metadata.sourceFiles()).hasSize(1); assertThat(metadata.sourceFiles().get(0).fileName()).isEqualTo("data-1"); @@ -125,8 +125,8 @@ void testBuildSkipsNullAndSnapshotDeletedRows() throws Exception { "test-vector-ann"); } - PkVectorSegmentMeta metadata = - PkVectorSegmentMeta.deserialize(segment.globalIndexMeta().indexMeta()); + PkVectorAnnSegmentMeta metadata = + PkVectorAnnSegmentMeta.deserialize(segment.globalIndexMeta().indexMeta()); assertThat(segment.rowCount()).isEqualTo(1); } @@ -159,8 +159,8 @@ void testAnnSearchUsesRowPositionDeletionMask() throws Exception { "l2", "test-vector-ann"); } - PkVectorSegmentMeta metadata = - PkVectorSegmentMeta.deserialize(segment.globalIndexMeta().indexMeta()); + PkVectorAnnSegmentMeta metadata = + PkVectorAnnSegmentMeta.deserialize(segment.globalIndexMeta().indexMeta()); ExecutorService executor = Executors.newSingleThreadExecutor(); BitmapDeletionVector deletionVector = new BitmapDeletionVector(); deletionVector.delete(0); @@ -228,12 +228,12 @@ void testBuildsAndSearchesMultiSourceAnnSegment() throws Exception { "test-vector-ann"); } - PkVectorSegmentMeta metadata = - PkVectorSegmentMeta.deserialize(segment.globalIndexMeta().indexMeta()); + PkVectorAnnSegmentMeta metadata = + PkVectorAnnSegmentMeta.deserialize(segment.globalIndexMeta().indexMeta()); assertThat(metadata.ordinalLayout()) - .isEqualTo(PkVectorSegmentMeta.OrdinalLayout.FILE_POSITION); + .isEqualTo(PkVectorAnnSegmentMeta.OrdinalLayout.FILE_POSITION); assertThat(metadata.sourceFiles()) - .extracting(PkVectorSegmentMeta.SourceFile::fileName) + .extracting(PkVectorSourceFile::fileName) .containsExactly("data-1", "data-2"); BitmapDeletionVector data2Deletes = new BitmapDeletionVector(); diff --git a/paimon-core/src/test/java/org/apache/paimon/index/pkvector/PkVectorAnnSegmentMetaTest.java b/paimon-core/src/test/java/org/apache/paimon/index/pkvector/PkVectorAnnSegmentMetaTest.java new file mode 100644 index 000000000000..36051763ccbe --- /dev/null +++ b/paimon-core/src/test/java/org/apache/paimon/index/pkvector/PkVectorAnnSegmentMetaTest.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.index.pkvector; + +import org.apache.paimon.io.DataOutputSerializer; + +import org.junit.jupiter.api.Test; + +import java.util.Arrays; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/** Tests for {@link PkVectorAnnSegmentMeta}. */ +class PkVectorAnnSegmentMetaTest { + + @Test + void testRoundTrip() { + PkVectorAnnSegmentMeta metadata = + new PkVectorAnnSegmentMeta( + "index-definition", + Arrays.asList( + new PkVectorSourceFile("data-1", 100), + new PkVectorSourceFile("data-2", 50)), + PkVectorAnnSegmentMeta.OrdinalLayout.FILE_POSITION, + new byte[] {1, 2, 3}); + + PkVectorAnnSegmentMeta restored = PkVectorAnnSegmentMeta.deserialize(metadata.serialize()); + + assertThat(restored.indexDefinitionId()).isEqualTo("index-definition"); + assertThat(restored.sourceFiles()).isEqualTo(metadata.sourceFiles()); + assertThat(restored.ordinalLayout()) + .isEqualTo(PkVectorAnnSegmentMeta.OrdinalLayout.FILE_POSITION); + assertThat(restored.payloadMetadata()).containsExactly(1, 2, 3); + } + + @Test + void testRejectsTruncatedPayloadMetadata() throws Exception { + DataOutputSerializer output = new DataOutputSerializer(128); + output.writeInt(1); + output.writeUTF("index"); + output.writeInt(1); + output.writeUTF("data-1"); + output.writeLong(10); + output.writeByte(PkVectorAnnSegmentMeta.OrdinalLayout.FILE_POSITION.ordinal()); + output.writeInt(1); + + assertThatThrownBy(() -> PkVectorAnnSegmentMeta.deserialize(output.getCopyOfBuffer())) + .hasMessageContaining("Failed to deserialize ANN vector segment metadata"); + } +} diff --git a/paimon-core/src/test/java/org/apache/paimon/index/pkvector/PkVectorRawSegmentMetaTest.java b/paimon-core/src/test/java/org/apache/paimon/index/pkvector/PkVectorRawSegmentMetaTest.java new file mode 100644 index 000000000000..6f48105fa596 --- /dev/null +++ b/paimon-core/src/test/java/org/apache/paimon/index/pkvector/PkVectorRawSegmentMetaTest.java @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.index.pkvector; + +import org.junit.jupiter.api.Test; + +import java.util.Arrays; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/** Tests for {@link PkVectorRawSegmentMeta}. */ +class PkVectorRawSegmentMetaTest { + + @Test + void testRoundTrip() { + PkVectorRawSegmentMeta metadata = + new PkVectorRawSegmentMeta( + "index-definition", new PkVectorSourceFile("data-1", 100)); + + PkVectorRawSegmentMeta restored = PkVectorRawSegmentMeta.deserialize(metadata.serialize()); + + assertThat(restored.indexDefinitionId()).isEqualTo("index-definition"); + assertThat(restored.sourceFile()).isEqualTo(new PkVectorSourceFile("data-1", 100)); + } + + @Test + void testRejectsTrailingBytes() { + PkVectorRawSegmentMeta metadata = + new PkVectorRawSegmentMeta("index", new PkVectorSourceFile("data", 1)); + byte[] bytes = Arrays.copyOf(metadata.serialize(), metadata.serialize().length + 1); + + assertThatThrownBy(() -> PkVectorRawSegmentMeta.deserialize(bytes)) + .hasMessageContaining("Unexpected trailing bytes"); + } +} diff --git a/paimon-core/src/test/java/org/apache/paimon/index/pkvector/PkVectorSegmentMetaTest.java b/paimon-core/src/test/java/org/apache/paimon/index/pkvector/PkVectorSegmentMetaTest.java deleted file mode 100644 index ece36b55ebb6..000000000000 --- a/paimon-core/src/test/java/org/apache/paimon/index/pkvector/PkVectorSegmentMetaTest.java +++ /dev/null @@ -1,81 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.paimon.index.pkvector; - -import org.apache.paimon.io.DataOutputSerializer; - -import org.junit.jupiter.api.Test; - -import java.util.Arrays; - -import static org.apache.paimon.index.pkvector.PkVectorSegmentMeta.OrdinalLayout.FILE_POSITION; -import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.assertThatThrownBy; - -/** Tests for {@link PkVectorSegmentMeta}. */ -class PkVectorSegmentMetaTest { - - @Test - void testRoundTrip() { - PkVectorSegmentMeta metadata = - new PkVectorSegmentMeta( - "1d4502f1-9cf0-4d86-8d8d-5cc9ac05e108", - Arrays.asList( - new PkVectorSegmentMeta.SourceFile("data-1", 100), - new PkVectorSegmentMeta.SourceFile("data-2", 50)), - FILE_POSITION, - new byte[] {4, 5, 6}); - - PkVectorSegmentMeta restored = PkVectorSegmentMeta.deserialize(metadata.serialize()); - - assertThat(restored.indexDefinitionId()).isEqualTo(metadata.indexDefinitionId()); - assertThat(restored.sourceFiles()).isEqualTo(metadata.sourceFiles()); - assertThat(restored.ordinalLayout()).isEqualTo(FILE_POSITION); - assertThat(restored.payloadMetadata()).containsExactly(4, 5, 6); - } - - @Test - void testRejectTrailingBytes() { - PkVectorSegmentMeta metadata = - new PkVectorSegmentMeta( - "index", - Arrays.asList(new PkVectorSegmentMeta.SourceFile("data", 1)), - FILE_POSITION, - new byte[0]); - byte[] bytes = Arrays.copyOf(metadata.serialize(), metadata.serialize().length + 1); - - assertThatThrownBy(() -> PkVectorSegmentMeta.deserialize(bytes)) - .hasMessageContaining("Unexpected trailing bytes"); - } - - @Test - void testRejectsTruncatedPayloadMetadata() throws Exception { - DataOutputSerializer output = new DataOutputSerializer(128); - output.writeInt(1); - output.writeUTF("index"); - output.writeInt(1); - output.writeUTF("data-1"); - output.writeLong(10); - output.writeByte(FILE_POSITION.ordinal()); - output.writeInt(1); - - assertThatThrownBy(() -> PkVectorSegmentMeta.deserialize(output.getCopyOfBuffer())) - .hasMessageContaining("Failed to deserialize primary-key vector segment metadata"); - } -} From e671103d3d2307f362cef4428cf6798068f7358e Mon Sep 17 00:00:00 2001 From: JingsongLi Date: Sat, 11 Jul 2026 19:27:59 +0800 Subject: [PATCH 12/19] [core] Remove raw primary-key vector sidecars --- .../pkvector/PkVectorAnnSegmentFile.java | 55 ++-- .../pkvector/PkVectorRawSegmentFile.java | 170 ----------- .../pkvector/PkVectorRawSegmentMeta.java | 81 ------ .../pkvector/PkVectorReader.java} | 23 +- .../pkvector/RawVectorSidecarReader.java | 247 ---------------- .../pkvector/RawVectorSidecarWriter.java | 163 ----------- .../pkvector/PkVectorAnnSegmentFileTest.java | 274 ++++++------------ .../pkvector/PkVectorRawSegmentMetaTest.java | 52 ---- .../index/pkvector/RawVectorSidecarTest.java | 166 ----------- 9 files changed, 123 insertions(+), 1108 deletions(-) delete mode 100644 paimon-core/src/main/java/org/apache/paimon/index/pkvector/PkVectorRawSegmentFile.java delete mode 100644 paimon-core/src/main/java/org/apache/paimon/index/pkvector/PkVectorRawSegmentMeta.java rename paimon-core/src/main/java/org/apache/paimon/{io/KeyValueVectorSidecarWriter.java => index/pkvector/PkVectorReader.java} (58%) delete mode 100644 paimon-core/src/main/java/org/apache/paimon/index/pkvector/RawVectorSidecarReader.java delete mode 100644 paimon-core/src/main/java/org/apache/paimon/index/pkvector/RawVectorSidecarWriter.java delete mode 100644 paimon-core/src/test/java/org/apache/paimon/index/pkvector/PkVectorRawSegmentMetaTest.java delete mode 100644 paimon-core/src/test/java/org/apache/paimon/index/pkvector/RawVectorSidecarTest.java diff --git a/paimon-core/src/main/java/org/apache/paimon/index/pkvector/PkVectorAnnSegmentFile.java b/paimon-core/src/main/java/org/apache/paimon/index/pkvector/PkVectorAnnSegmentFile.java index c81625a34c5d..926256e4738b 100644 --- a/paimon-core/src/main/java/org/apache/paimon/index/pkvector/PkVectorAnnSegmentFile.java +++ b/paimon-core/src/main/java/org/apache/paimon/index/pkvector/PkVectorAnnSegmentFile.java @@ -102,35 +102,34 @@ public IndexFileMeta build( long fileOffset = 0; int dimension = -1; for (Source source : sources) { - RawVectorSidecarReader rawVectors = source.openReader(); + PkVectorReader vectors = source.openReader(); try { checkArgument( - rawVectors.rowCount() == source.sourceFile.rowCount(), - "Raw vector row count %s does not match source file %s row count %s.", - rawVectors.rowCount(), + vectors.rowCount() == source.sourceFile.rowCount(), + "Vector row count %s does not match source file %s row count %s.", + vectors.rowCount(), source.sourceFile.fileName(), source.sourceFile.rowCount()); if (dimension < 0) { - dimension = rawVectors.dimension(); + dimension = vectors.dimension(); } checkArgument( - rawVectors.dimension() == dimension, - "Raw vector source %s dimension %s does not match dimension %s.", + vectors.dimension() == dimension, + "Vector source %s dimension %s does not match dimension %s.", source.sourceFile.fileName(), - rawVectors.dimension(), + vectors.dimension(), dimension); if (vectorField.type() instanceof VectorType) { checkArgument( ((VectorType) vectorField.type()).getLength() == dimension, - "Vector field dimension %s does not match raw vector dimension %s.", + "Vector field dimension %s does not match source vector dimension %s.", ((VectorType) vectorField.type()).getLength(), dimension); } float[] vector = new float[dimension]; - rawVectors.rewind(); - for (long rowPosition = 0; rowPosition < rawVectors.rowCount(); rowPosition++) { - boolean present = rawVectors.readNextVector(vector); + for (long rowPosition = 0; rowPosition < vectors.rowCount(); rowPosition++) { + boolean present = vectors.readNextVector(vector); if (!present || source.excludedPosition.test(rowPosition)) { continue; } @@ -138,7 +137,7 @@ public IndexFileMeta build( liveRowCount++; } } finally { - source.closeReader(rawVectors); + source.closeReader(vectors); } fileOffset += source.sourceFile.rowCount(); } @@ -212,31 +211,29 @@ private void deleteCreatedFiles() { } } - /** One raw vector source used while building an ANN segment. */ + /** One vector source used while building an ANN segment. */ public static class Source { private final PkVectorSourceFile sourceFile; - @Nullable private final RawVectorSidecarReader rawVectors; + @Nullable private final PkVectorReader vectors; @Nullable private final ReaderFactory readerFactory; private final LongPredicate excludedPosition; - public Source(DataFileMeta sourceFile, RawVectorSidecarReader rawVectors) { - this(sourceFile, rawVectors, position -> false); + public Source(DataFileMeta sourceFile, PkVectorReader vectors) { + this(sourceFile, vectors, position -> false); } public Source( - DataFileMeta sourceFile, - RawVectorSidecarReader rawVectors, - LongPredicate excludedPosition) { - this(sourceMetadata(sourceFile), rawVectors, excludedPosition); + DataFileMeta sourceFile, PkVectorReader vectors, LongPredicate excludedPosition) { + this(sourceMetadata(sourceFile), vectors, excludedPosition); } Source( PkVectorSourceFile sourceFile, - RawVectorSidecarReader rawVectors, + PkVectorReader vectors, LongPredicate excludedPosition) { this.sourceFile = sourceFile; - this.rawVectors = rawVectors; + this.vectors = vectors; this.readerFactory = null; this.excludedPosition = excludedPosition; } @@ -246,7 +243,7 @@ private Source( ReaderFactory readerFactory, LongPredicate excludedPosition) { this.sourceFile = sourceFile; - this.rawVectors = null; + this.vectors = null; this.readerFactory = readerFactory; this.excludedPosition = excludedPosition; } @@ -255,19 +252,19 @@ static Source lazy(PkVectorSourceFile sourceFile, ReaderFactory readerFactory) { return new Source(sourceFile, readerFactory, position -> false); } - private RawVectorSidecarReader openReader() throws IOException { - return rawVectors != null ? rawVectors : readerFactory.open(); + private PkVectorReader openReader() throws IOException { + return vectors != null ? vectors : readerFactory.open(); } - private void closeReader(RawVectorSidecarReader reader) throws IOException { - if (rawVectors == null) { + private void closeReader(PkVectorReader reader) throws IOException { + if (vectors == null) { reader.close(); } } @FunctionalInterface interface ReaderFactory { - RawVectorSidecarReader open() throws IOException; + PkVectorReader open() throws IOException; } } } diff --git a/paimon-core/src/main/java/org/apache/paimon/index/pkvector/PkVectorRawSegmentFile.java b/paimon-core/src/main/java/org/apache/paimon/index/pkvector/PkVectorRawSegmentFile.java deleted file mode 100644 index d0a3a6ffe2f4..000000000000 --- a/paimon-core/src/main/java/org/apache/paimon/index/pkvector/PkVectorRawSegmentFile.java +++ /dev/null @@ -1,170 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.paimon.index.pkvector; - -import org.apache.paimon.data.InternalArray; -import org.apache.paimon.fs.FileIO; -import org.apache.paimon.fs.Path; -import org.apache.paimon.index.GlobalIndexMeta; -import org.apache.paimon.index.IndexFile; -import org.apache.paimon.index.IndexFileMeta; -import org.apache.paimon.index.IndexPathFactory; -import org.apache.paimon.io.DataFileMeta; -import org.apache.paimon.io.FileWriterAbortExecutor; -import org.apache.paimon.io.KeyValueVectorSidecarWriter; -import org.apache.paimon.manifest.FileSource; - -import javax.annotation.Nullable; - -import java.io.IOException; -import java.io.UncheckedIOException; -import java.util.Optional; -import java.util.function.BiConsumer; -import java.util.function.Consumer; - -import static org.apache.paimon.utils.Preconditions.checkArgument; -import static org.apache.paimon.utils.Preconditions.checkState; - -/** Creates immutable row-position-addressable raw vector segments for data files. */ -public class PkVectorRawSegmentFile extends IndexFile { - - public static final String PK_VECTOR_RAW = "pk-vector-raw"; - - public PkVectorRawSegmentFile(FileIO fileIO, IndexPathFactory pathFactory) { - super(fileIO, pathFactory); - } - - RawVectorSidecarReader newReader(IndexFileMeta segment) throws IOException { - return new RawVectorSidecarReader(fileIO, path(segment)); - } - - public KeyValueVectorSidecarWriter newWriter( - int dimension, - String indexDefinitionId, - int vectorFieldId, - BiConsumer segmentConsumer, - Consumer segmentAbortConsumer) { - Path path = pathFactory.newPath(); - try { - return new Writer( - new RawVectorSidecarWriter(fileIO, path, dimension), - indexDefinitionId, - vectorFieldId, - segmentConsumer, - segmentAbortConsumer); - } catch (IOException e) { - fileIO.deleteQuietly(path); - throw new UncheckedIOException( - "Failed to create primary-key raw vector segment: " + path, e); - } - } - - private class Writer implements KeyValueVectorSidecarWriter { - - private final RawVectorSidecarWriter rawWriter; - private final String indexDefinitionId; - private final int vectorFieldId; - private final BiConsumer segmentConsumer; - private final Consumer segmentAbortConsumer; - - @Nullable private IndexFileMeta completedSegment; - private boolean closed; - private boolean completed; - - private Writer( - RawVectorSidecarWriter rawWriter, - String indexDefinitionId, - int vectorFieldId, - BiConsumer segmentConsumer, - Consumer segmentAbortConsumer) { - this.rawWriter = rawWriter; - this.indexDefinitionId = indexDefinitionId; - this.vectorFieldId = vectorFieldId; - this.segmentConsumer = segmentConsumer; - this.segmentAbortConsumer = segmentAbortConsumer; - } - - @Override - public void write(@Nullable InternalArray vector) throws IOException { - rawWriter.write(vector); - } - - @Override - public void close() throws IOException { - if (!closed) { - rawWriter.close(); - closed = true; - } - } - - @Override - public void complete(DataFileMeta sourceFile) throws IOException { - checkState(closed, "Raw vector segment must be closed before completion."); - checkState(!completed, "Raw vector segment is already completed."); - checkArgument( - rawWriter.rowCount() == sourceFile.rowCount(), - "Raw vector segment row count %s does not match source file %s row count %s.", - rawWriter.rowCount(), - sourceFile.fileName(), - sourceFile.rowCount()); - - PkVectorRawSegmentMeta metadata = - new PkVectorRawSegmentMeta( - indexDefinitionId, - new PkVectorSourceFile(sourceFile.fileName(), sourceFile.rowCount())); - Path path = rawWriter.path(); - IndexFileMeta segment = - new IndexFileMeta( - PK_VECTOR_RAW, - path.getName(), - fileIO.getFileSize(path), - rawWriter.liveVectorCount(), - new GlobalIndexMeta( - 0, - sourceFile.rowCount(), - vectorFieldId, - null, - metadata.serialize()), - pathFactory.isExternalPath() ? path.toString() : null); - segmentConsumer.accept(segment, sourceFile.fileSource().orElse(FileSource.APPEND)); - completedSegment = segment; - completed = true; - } - - @Override - public void abort() { - if (completedSegment != null) { - segmentAbortConsumer.accept(completedSegment); - completedSegment = null; - } - rawWriter.abort(); - } - - @Override - public Optional abortExecutor() { - return Optional.of( - new FileWriterAbortExecutor(fileIO, rawWriter.path()) { - @Override - public void abort() { - Writer.this.abort(); - } - }); - } - } -} diff --git a/paimon-core/src/main/java/org/apache/paimon/index/pkvector/PkVectorRawSegmentMeta.java b/paimon-core/src/main/java/org/apache/paimon/index/pkvector/PkVectorRawSegmentMeta.java deleted file mode 100644 index 8b4e63ee9a93..000000000000 --- a/paimon-core/src/main/java/org/apache/paimon/index/pkvector/PkVectorRawSegmentMeta.java +++ /dev/null @@ -1,81 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.paimon.index.pkvector; - -import org.apache.paimon.io.DataInputDeserializer; -import org.apache.paimon.io.DataOutputSerializer; - -import java.io.IOException; -import java.util.Objects; - -import static org.apache.paimon.utils.Preconditions.checkArgument; - -/** Versioned metadata for an immutable RAW primary-key vector segment. */ -public final class PkVectorRawSegmentMeta { - - private static final int VERSION = 1; - - private final String indexDefinitionId; - private final PkVectorSourceFile sourceFile; - - public PkVectorRawSegmentMeta(String indexDefinitionId, PkVectorSourceFile sourceFile) { - this.indexDefinitionId = Objects.requireNonNull(indexDefinitionId); - this.sourceFile = Objects.requireNonNull(sourceFile); - } - - public String indexDefinitionId() { - return indexDefinitionId; - } - - public PkVectorSourceFile sourceFile() { - return sourceFile; - } - - public byte[] serialize() { - try { - DataOutputSerializer output = new DataOutputSerializer(64); - output.writeInt(VERSION); - output.writeUTF(indexDefinitionId); - output.writeUTF(sourceFile.fileName()); - output.writeLong(sourceFile.rowCount()); - return output.getCopyOfBuffer(); - } catch (IOException e) { - throw new RuntimeException("Failed to serialize RAW vector segment metadata.", e); - } - } - - public static PkVectorRawSegmentMeta deserialize(byte[] bytes) { - try { - DataInputDeserializer input = new DataInputDeserializer(bytes); - int version = input.readInt(); - checkArgument( - version == VERSION, "Unsupported RAW vector segment version: %s.", version); - String indexDefinitionId = input.readUTF(); - PkVectorSourceFile sourceFile = - new PkVectorSourceFile(input.readUTF(), input.readLong()); - checkArgument( - input.available() == 0, - "Unexpected trailing bytes in RAW vector segment metadata."); - return new PkVectorRawSegmentMeta(indexDefinitionId, sourceFile); - } catch (IOException e) { - throw new IllegalArgumentException( - "Failed to deserialize RAW vector segment metadata.", e); - } - } -} diff --git a/paimon-core/src/main/java/org/apache/paimon/io/KeyValueVectorSidecarWriter.java b/paimon-core/src/main/java/org/apache/paimon/index/pkvector/PkVectorReader.java similarity index 58% rename from paimon-core/src/main/java/org/apache/paimon/io/KeyValueVectorSidecarWriter.java rename to paimon-core/src/main/java/org/apache/paimon/index/pkvector/PkVectorReader.java index 11bd426f21c8..b173607ce082 100644 --- a/paimon-core/src/main/java/org/apache/paimon/io/KeyValueVectorSidecarWriter.java +++ b/paimon-core/src/main/java/org/apache/paimon/index/pkvector/PkVectorReader.java @@ -16,27 +16,18 @@ * limitations under the License. */ -package org.apache.paimon.io; - -import org.apache.paimon.data.InternalArray; - -import javax.annotation.Nullable; +package org.apache.paimon.index.pkvector; import java.io.Closeable; import java.io.IOException; -import java.util.Optional; - -/** Synchronous vector sidecar owned by one key-value data-file writer. */ -public interface KeyValueVectorSidecarWriter extends Closeable { - void write(@Nullable InternalArray vector) throws IOException; +/** Sequential vectors in physical data-file row order. */ +public interface PkVectorReader extends Closeable { - void complete(DataFileMeta sourceFile) throws IOException; + int dimension(); - void abort(); + long rowCount(); - /** Lightweight cleanup retained by a rolling writer after this sidecar is completed. */ - default Optional abortExecutor() { - return Optional.empty(); - } + /** Reads the next physical row and returns whether it contains a non-null vector. */ + boolean readNextVector(float[] reuse) throws IOException; } diff --git a/paimon-core/src/main/java/org/apache/paimon/index/pkvector/RawVectorSidecarReader.java b/paimon-core/src/main/java/org/apache/paimon/index/pkvector/RawVectorSidecarReader.java deleted file mode 100644 index 7d77c05fe816..000000000000 --- a/paimon-core/src/main/java/org/apache/paimon/index/pkvector/RawVectorSidecarReader.java +++ /dev/null @@ -1,247 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.paimon.index.pkvector; - -import org.apache.paimon.fs.FileIO; -import org.apache.paimon.fs.Path; -import org.apache.paimon.fs.SeekableInputStream; - -import java.io.Closeable; -import java.io.DataInputStream; -import java.io.IOException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.Comparator; -import java.util.List; -import java.util.PriorityQueue; -import java.util.function.LongPredicate; - -import static org.apache.paimon.index.pkvector.RawVectorSidecarWriter.HEADER_SIZE; -import static org.apache.paimon.index.pkvector.RawVectorSidecarWriter.MAGIC; -import static org.apache.paimon.index.pkvector.RawVectorSidecarWriter.VERSION; -import static org.apache.paimon.index.pkvector.RawVectorSidecarWriter.recordSize; -import static org.apache.paimon.utils.Preconditions.checkArgument; - -/** Random-access reader for raw vector sidecars. This class is not thread-safe. */ -public class RawVectorSidecarReader implements Closeable { - - private static final Comparator BEST_FIRST = - (left, right) -> { - int distance = Float.compare(left.distance, right.distance); - return distance != 0 ? distance : Long.compare(left.rowPosition, right.rowPosition); - }; - - private final SeekableInputStream input; - private final DataInputStream dataInput; - private final int dimension; - private final int recordSize; - private final long rowCount; - private long nextSequentialPosition; - - public RawVectorSidecarReader(FileIO fileIO, Path path) throws IOException { - long fileSize = fileIO.getFileSize(path); - checkArgument( - fileSize >= HEADER_SIZE, "Raw vector sidecar %s is shorter than its header.", path); - this.input = fileIO.newInputStream(path); - this.dataInput = new DataInputStream(input); - int magic = dataInput.readInt(); - checkArgument(magic == MAGIC, "File %s is not a raw vector sidecar.", path); - int version = dataInput.readInt(); - checkArgument(version == VERSION, "Unsupported raw vector sidecar version: %s.", version); - this.dimension = dataInput.readInt(); - this.recordSize = dataInput.readInt(); - checkArgument(dimension > 0, "Raw vector sidecar dimension must be positive."); - checkArgument( - recordSize == recordSize(dimension), - "Raw vector sidecar record size %s does not match dimension %s.", - recordSize, - dimension); - long payloadSize = fileSize - HEADER_SIZE; - checkArgument( - payloadSize % recordSize == 0, - "Raw vector sidecar %s has a truncated record.", - path); - this.rowCount = payloadSize / recordSize; - this.nextSequentialPosition = 0; - } - - public int dimension() { - return dimension; - } - - public long rowCount() { - return rowCount; - } - - /** Raw sidecars use row positions directly as segment ordinals. */ - public long rowPositionForOrdinal(long ordinal) { - checkRowPosition(ordinal); - return ordinal; - } - - public float[] readVector(long rowPosition) throws IOException { - checkRowPosition(rowPosition); - nextSequentialPosition = -1; - input.seek(HEADER_SIZE + rowPosition * recordSize); - if (!dataInput.readBoolean()) { - return null; - } - float[] vector = new float[dimension]; - for (int i = 0; i < dimension; i++) { - vector[i] = dataInput.readFloat(); - } - return vector; - } - - /** Positions this reader for a sequential pass over all row-position records. */ - public void rewind() throws IOException { - input.seek(HEADER_SIZE); - nextSequentialPosition = 0; - } - - /** - * Reads the next record into a caller-owned reusable buffer and returns whether it is non-null. - */ - public boolean readNextVector(float[] reuse) throws IOException { - checkArgument( - reuse.length == dimension, - "Reusable vector buffer dimension must be %s, but is %s.", - dimension, - reuse.length); - checkArgument( - nextSequentialPosition >= 0, - "Raw vector sequential read requires rewind after random access."); - checkArgument( - nextSequentialPosition < rowCount, - "No raw vector remains after row position %s.", - nextSequentialPosition); - boolean present = dataInput.readBoolean(); - for (int i = 0; i < dimension; i++) { - reuse[i] = dataInput.readFloat(); - } - nextSequentialPosition++; - return present; - } - - /** Performs an exact top-k scan and excludes positions deleted in the selected snapshot. */ - public List search( - float[] query, String metric, int limit, LongPredicate excludedPosition) - throws IOException { - checkArgument(query.length == dimension, "Query vector dimension must be %s.", dimension); - checkArgument(limit > 0, "Vector search limit must be positive: %s.", limit); - checkArgument( - "l2".equals(metric) || "cosine".equals(metric) || "inner_product".equals(metric), - "Unsupported raw vector distance metric: %s.", - metric); - for (int i = 0; i < query.length; i++) { - checkArgument( - !Float.isNaN(query[i]) && !Float.isInfinite(query[i]), - "Query vector element at index %s must be finite.", - i); - } - - PriorityQueue nearest = new PriorityQueue<>(limit, BEST_FIRST.reversed()); - float[] vector = new float[dimension]; - nextSequentialPosition = -1; - input.seek(HEADER_SIZE); - for (long rowPosition = 0; rowPosition < rowCount; rowPosition++) { - boolean present = dataInput.readBoolean(); - for (int i = 0; i < dimension; i++) { - vector[i] = dataInput.readFloat(); - } - if (!present || excludedPosition.test(rowPosition)) { - continue; - } - - Candidate candidate = new Candidate(rowPosition, distance(query, vector, metric)); - if (nearest.size() < limit) { - nearest.add(candidate); - } else if (BEST_FIRST.compare(candidate, nearest.peek()) < 0) { - nearest.poll(); - nearest.add(candidate); - } - } - - List result = new ArrayList<>(nearest); - Collections.sort(result, BEST_FIRST); - return result; - } - - @Override - public void close() throws IOException { - dataInput.close(); - } - - private void checkRowPosition(long rowPosition) { - checkArgument( - rowPosition >= 0 && rowPosition < rowCount, - "Raw vector row position %s is outside [0, %s).", - rowPosition, - rowCount); - } - - private float distance(float[] query, float[] vector, String metric) { - if ("l2".equals(metric)) { - double squaredDistance = 0; - for (int i = 0; i < dimension; i++) { - double delta = vector[i] - query[i]; - squaredDistance += delta * delta; - } - return (float) squaredDistance; - } - - double dot = 0; - double queryNorm = 0; - double vectorNorm = 0; - for (int i = 0; i < dimension; i++) { - dot += query[i] * vector[i]; - queryNorm += query[i] * query[i]; - vectorNorm += vector[i] * vector[i]; - } - if ("inner_product".equals(metric)) { - return (float) -dot; - } - if (queryNorm == 0 || vectorNorm == 0) { - return 1; - } - double similarity = dot / Math.sqrt(queryNorm * vectorNorm); - similarity = Math.max(-1, Math.min(1, similarity)); - return (float) (1 - similarity); - } - - /** One exact raw-vector candidate. Lower distance is better. */ - public static class Candidate { - - private final long rowPosition; - private final float distance; - - private Candidate(long rowPosition, float distance) { - this.rowPosition = rowPosition; - this.distance = distance; - } - - public long rowPosition() { - return rowPosition; - } - - public float distance() { - return distance; - } - } -} diff --git a/paimon-core/src/main/java/org/apache/paimon/index/pkvector/RawVectorSidecarWriter.java b/paimon-core/src/main/java/org/apache/paimon/index/pkvector/RawVectorSidecarWriter.java deleted file mode 100644 index 27ad85e2670e..000000000000 --- a/paimon-core/src/main/java/org/apache/paimon/index/pkvector/RawVectorSidecarWriter.java +++ /dev/null @@ -1,163 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.paimon.index.pkvector; - -import org.apache.paimon.data.InternalArray; -import org.apache.paimon.fs.FileIO; -import org.apache.paimon.fs.Path; -import org.apache.paimon.fs.PositionOutputStream; - -import javax.annotation.Nullable; - -import java.io.Closeable; -import java.io.DataOutputStream; -import java.io.IOException; - -import static org.apache.paimon.utils.Preconditions.checkArgument; -import static org.apache.paimon.utils.Preconditions.checkState; - -/** Streaming writer for a row-position-addressable raw float-vector sidecar. */ -public class RawVectorSidecarWriter implements Closeable { - - static final int MAGIC = 0x50565231; - static final int VERSION = 1; - static final int HEADER_SIZE = 16; - - private final FileIO fileIO; - private final Path path; - private final int dimension; - private final DataOutputStream output; - private final float[] vectorBuffer; - - private long rowCount; - private long liveVectorCount; - private boolean closed; - - public RawVectorSidecarWriter(FileIO fileIO, Path path, int dimension) throws IOException { - checkArgument(dimension > 0, "Raw vector dimension must be positive: %s.", dimension); - checkArgument( - dimension <= (Integer.MAX_VALUE - 1) / Float.BYTES, - "Raw vector dimension is too large: %s.", - dimension); - this.fileIO = fileIO; - this.path = path; - this.dimension = dimension; - this.vectorBuffer = new float[dimension]; - PositionOutputStream stream = fileIO.newOutputStream(path, false); - this.output = new DataOutputStream(stream); - this.output.writeInt(MAGIC); - this.output.writeInt(VERSION); - this.output.writeInt(dimension); - this.output.writeInt(recordSize(dimension)); - } - - public void write(@Nullable Object vector) throws IOException { - checkState(!closed, "Raw vector sidecar writer is already closed."); - if (vector == null) { - output.writeBoolean(false); - for (int i = 0; i < dimension; i++) { - output.writeFloat(0); - } - rowCount++; - return; - } - - float[] values = materializeAndValidate(vector); - output.writeBoolean(true); - for (float value : values) { - output.writeFloat(value); - } - rowCount++; - liveVectorCount++; - } - - public long rowCount() { - return rowCount; - } - - public long liveVectorCount() { - return liveVectorCount; - } - - public Path path() { - return path; - } - - @Override - public void close() throws IOException { - if (!closed) { - closed = true; - output.close(); - } - } - - /** Best-effort cleanup used when the owning data-file writer fails or is aborted. */ - public void abort() { - try { - close(); - } catch (IOException ignored) { - // Keep abort best-effort and always try to remove the incomplete sidecar. - } - fileIO.deleteQuietly(path); - } - - static int recordSize(int dimension) { - return 1 + dimension * Float.BYTES; - } - - private void checkDimension(int actualDimension) { - checkArgument( - actualDimension == dimension, - "Raw vector dimension must be %s, but was %s.", - dimension, - actualDimension); - } - - private float[] materializeAndValidate(Object vector) { - if (vector instanceof float[]) { - float[] values = (float[]) vector; - checkDimension(values.length); - for (int i = 0; i < dimension; i++) { - checkFinite(values[i], i); - } - return values; - } - if (vector instanceof InternalArray) { - InternalArray values = (InternalArray) vector; - checkDimension(values.size()); - for (int i = 0; i < dimension; i++) { - checkArgument(!values.isNullAt(i), "Vector element at index %s is null.", i); - float value = values.getFloat(i); - checkFinite(value, i); - vectorBuffer[i] = value; - } - return vectorBuffer; - } - throw new IllegalArgumentException( - "Unsupported raw vector value type: " + vector.getClass().getName()); - } - - private static void checkFinite(float value, int index) { - checkArgument( - !Float.isNaN(value) && !Float.isInfinite(value), - "Vector element at index %s must be finite, but was %s.", - index, - value); - } -} diff --git a/paimon-core/src/test/java/org/apache/paimon/index/pkvector/PkVectorAnnSegmentFileTest.java b/paimon-core/src/test/java/org/apache/paimon/index/pkvector/PkVectorAnnSegmentFileTest.java index 03138e81eea2..106d24716f1d 100644 --- a/paimon-core/src/test/java/org/apache/paimon/index/pkvector/PkVectorAnnSegmentFileTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/index/pkvector/PkVectorAnnSegmentFileTest.java @@ -33,6 +33,7 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; +import java.io.IOException; import java.util.Arrays; import java.util.Collections; import java.util.HashMap; @@ -46,201 +47,67 @@ import static org.apache.paimon.index.pkvector.PkVectorAnnSegmentMeta.OrdinalLayout.ROW_POSITION; import static org.assertj.core.api.Assertions.assertThat; -/** Tests ANN payload construction through the vector GlobalIndexer SPI. */ +/** Tests ANN construction from generic vector readers. */ class PkVectorAnnSegmentFileTest { @TempDir java.nio.file.Path tempPath; @Test - void testBuildsSingleSourceAnnSegmentWithRowPositionOrdinals() throws Exception { + void testBuildSkipsNullAndExcludedPhysicalRows() throws Exception { LocalFileIO fileIO = LocalFileIO.create(); - IndexPathFactory pathFactory = pathFactory(); - Path rawPath = new Path(tempPath.resolve("raw").toUri()); - try (RawVectorSidecarWriter writer = new RawVectorSidecarWriter(fileIO, rawPath, 2)) { - writer.write(new float[] {0, 0}); - writer.write(new float[] {2, 0}); - } - Options options = new Options(); - options.setString("test.vector.dimension", "2"); - options.setString("test.vector.metric", "l2"); - DataField vectorField = new DataField(7, "embedding", DataTypes.ARRAY(DataTypes.FLOAT())); - - IndexFileMeta segment; - try (RawVectorSidecarReader rawReader = new RawVectorSidecarReader(fileIO, rawPath)) { - segment = - new PkVectorAnnSegmentFile(fileIO, pathFactory) - .build( - Collections.singletonList( - new PkVectorAnnSegmentFile.Source( - dataFile("data-1"), rawReader)), - ROW_POSITION, - vectorField, - options, - "definition", - "l2", - "test-vector-ann"); - } + IndexFileMeta segment = + annFile(fileIO) + .build( + Collections.singletonList( + new PkVectorAnnSegmentFile.Source( + dataFile("data-1", 3), + new ArrayReader( + new float[][] {{0, 0}, null, {2, 0}}), + position -> position == 0)), + ROW_POSITION, + vectorField(), + indexOptions(), + "definition", + "l2", + "test-vector-ann"); assertThat(segment.indexType()).isEqualTo(PkVectorAnnSegmentFile.PK_VECTOR_ANN); - assertThat(segment.rowCount()).isEqualTo(2); - assertThat(fileIO.exists(pathFactory.toPath(segment))).isTrue(); - PkVectorAnnSegmentMeta metadata = - PkVectorAnnSegmentMeta.deserialize(segment.globalIndexMeta().indexMeta()); - assertThat(metadata.ordinalLayout()).isEqualTo(ROW_POSITION); - assertThat(metadata.sourceFiles()).hasSize(1); - assertThat(metadata.sourceFiles().get(0).fileName()).isEqualTo("data-1"); - assertThat(segment.globalIndexMeta().indexFieldId()).isEqualTo(7); - } - - @Test - void testBuildSkipsNullAndSnapshotDeletedRows() throws Exception { - LocalFileIO fileIO = LocalFileIO.create(); - IndexPathFactory pathFactory = pathFactory(); - Path rawPath = new Path(tempPath.resolve("raw-with-deletes").toUri()); - try (RawVectorSidecarWriter writer = new RawVectorSidecarWriter(fileIO, rawPath, 2)) { - writer.write(new float[] {0, 0}); - writer.write(null); - writer.write(new float[] {2, 0}); - } - Options options = new Options(); - options.setString("test.vector.dimension", "2"); - options.setString("test.vector.metric", "l2"); - DataField vectorField = new DataField(7, "embedding", DataTypes.ARRAY(DataTypes.FLOAT())); - - IndexFileMeta segment; - try (RawVectorSidecarReader rawReader = new RawVectorSidecarReader(fileIO, rawPath)) { - segment = - new PkVectorAnnSegmentFile(fileIO, pathFactory) - .build( - Collections.singletonList( - new PkVectorAnnSegmentFile.Source( - dataFile("data-1", 3), - rawReader, - position -> position == 0)), - ROW_POSITION, - vectorField, - options, - "definition", - "l2", - "test-vector-ann"); - } - - PkVectorAnnSegmentMeta metadata = - PkVectorAnnSegmentMeta.deserialize(segment.globalIndexMeta().indexMeta()); assertThat(segment.rowCount()).isEqualTo(1); - } - - @Test - void testAnnSearchUsesRowPositionDeletionMask() throws Exception { - LocalFileIO fileIO = LocalFileIO.create(); - IndexPathFactory pathFactory = pathFactory(); - Path rawPath = new Path(tempPath.resolve("ann-search-raw").toUri()); - try (RawVectorSidecarWriter writer = new RawVectorSidecarWriter(fileIO, rawPath, 2)) { - writer.write(new float[] {0, 0}); - writer.write(new float[] {1, 0}); - writer.write(new float[] {2, 0}); - } - Options options = new Options(); - options.setString("test.vector.dimension", "2"); - options.setString("test.vector.metric", "l2"); - DataField vectorField = new DataField(7, "embedding", DataTypes.ARRAY(DataTypes.FLOAT())); - PkVectorAnnSegmentFile annFile = new PkVectorAnnSegmentFile(fileIO, pathFactory); - IndexFileMeta segment; - try (RawVectorSidecarReader rawReader = new RawVectorSidecarReader(fileIO, rawPath)) { - segment = - annFile.build( - Collections.singletonList( - new PkVectorAnnSegmentFile.Source( - dataFile("data-1", 3), rawReader)), - ROW_POSITION, - vectorField, - options, - "definition", - "l2", - "test-vector-ann"); - } PkVectorAnnSegmentMeta metadata = PkVectorAnnSegmentMeta.deserialize(segment.globalIndexMeta().indexMeta()); - ExecutorService executor = Executors.newSingleThreadExecutor(); - BitmapDeletionVector deletionVector = new BitmapDeletionVector(); - deletionVector.delete(0); - List candidates; - try { - candidates = - new PkVectorAnnSegmentSearcher( - fileIO, - annFile, - vectorField, - options, - "test-vector-ann", - "l2", - executor) - .search( - segment, - metadata, - new float[] {0, 0}, - 2, - deletionVector, - Collections.emptyMap()); - } finally { - executor.shutdownNow(); - } - - assertThat(candidates) - .extracting(PkVectorAnnSegmentSearcher.Candidate::rowPosition) - .containsExactly(1L, 2L); - assertThat(candidates) - .extracting(PkVectorAnnSegmentSearcher.Candidate::distance) - .containsExactly(1F, 4F); + assertThat(metadata.ordinalLayout()).isEqualTo(ROW_POSITION); + assertThat(metadata.sourceFiles()) + .extracting(PkVectorSourceFile::fileName) + .containsExactly("data-1"); } @Test - void testBuildsAndSearchesMultiSourceAnnSegment() throws Exception { + void testBuildsAndSearchesMultiSourceSegment() throws Exception { LocalFileIO fileIO = LocalFileIO.create(); - IndexPathFactory pathFactory = pathFactory(); - Path raw1Path = new Path(tempPath.resolve("multi-raw-1").toUri()); - Path raw2Path = new Path(tempPath.resolve("multi-raw-2").toUri()); - try (RawVectorSidecarWriter writer1 = new RawVectorSidecarWriter(fileIO, raw1Path, 2); - RawVectorSidecarWriter writer2 = new RawVectorSidecarWriter(fileIO, raw2Path, 2)) { - writer1.write(new float[] {5, 0}); - writer1.write(new float[] {10, 0}); - writer2.write(new float[] {0, 0}); - writer2.write(new float[] {2, 0}); - } - Options options = new Options(); - options.setString("test.vector.dimension", "2"); - options.setString("test.vector.metric", "l2"); - DataField vectorField = new DataField(7, "embedding", DataTypes.ARRAY(DataTypes.FLOAT())); - PkVectorAnnSegmentFile annFile = new PkVectorAnnSegmentFile(fileIO, pathFactory); - IndexFileMeta segment; - try (RawVectorSidecarReader raw1 = new RawVectorSidecarReader(fileIO, raw1Path); - RawVectorSidecarReader raw2 = new RawVectorSidecarReader(fileIO, raw2Path)) { - segment = - annFile.build( - Arrays.asList( - new PkVectorAnnSegmentFile.Source(dataFile("data-1"), raw1), - new PkVectorAnnSegmentFile.Source(dataFile("data-2"), raw2)), - FILE_POSITION, - vectorField, - options, - "definition", - "l2", - "test-vector-ann"); - } - + PkVectorAnnSegmentFile annFile = annFile(fileIO); + IndexFileMeta segment = + annFile.build( + Arrays.asList( + new PkVectorAnnSegmentFile.Source( + dataFile("data-1", 2), + new ArrayReader(new float[][] {{5, 0}, {10, 0}})), + new PkVectorAnnSegmentFile.Source( + dataFile("data-2", 2), + new ArrayReader(new float[][] {{0, 0}, {2, 0}}))), + FILE_POSITION, + vectorField(), + indexOptions(), + "definition", + "l2", + "test-vector-ann"); PkVectorAnnSegmentMeta metadata = PkVectorAnnSegmentMeta.deserialize(segment.globalIndexMeta().indexMeta()); - assertThat(metadata.ordinalLayout()) - .isEqualTo(PkVectorAnnSegmentMeta.OrdinalLayout.FILE_POSITION); - assertThat(metadata.sourceFiles()) - .extracting(PkVectorSourceFile::fileName) - .containsExactly("data-1", "data-2"); - BitmapDeletionVector data2Deletes = new BitmapDeletionVector(); data2Deletes.delete(0); Map deletionVectors = new HashMap<>(); deletionVectors.put("data-2", data2Deletes); + ExecutorService executor = Executors.newSingleThreadExecutor(); List candidates; try { @@ -248,8 +115,8 @@ void testBuildsAndSearchesMultiSourceAnnSegment() throws Exception { new PkVectorAnnSegmentSearcher( fileIO, annFile, - vectorField, - options, + vectorField(), + indexOptions(), "test-vector-ann", "l2", executor) @@ -264,6 +131,7 @@ void testBuildsAndSearchesMultiSourceAnnSegment() throws Exception { executor.shutdownNow(); } + assertThat(metadata.ordinalLayout()).isEqualTo(FILE_POSITION); assertThat(candidates) .extracting( PkVectorAnnSegmentSearcher.Candidate::dataFileName, @@ -272,16 +140,21 @@ void testBuildsAndSearchesMultiSourceAnnSegment() throws Exception { org.assertj.core.groups.Tuple.tuple("data-2", 1L), org.assertj.core.groups.Tuple.tuple("data-1", 0L), org.assertj.core.groups.Tuple.tuple("data-1", 1L)); - assertThat(candidates.get(0).distance()) - .isCloseTo(4F, org.assertj.core.data.Offset.offset(0.001F)); - assertThat(candidates.get(1).distance()) - .isCloseTo(25F, org.assertj.core.data.Offset.offset(0.001F)); - assertThat(candidates.get(2).distance()) - .isCloseTo(100F, org.assertj.core.data.Offset.offset(0.001F)); } - private static DataFileMeta dataFile(String fileName) { - return dataFile(fileName, 2); + private PkVectorAnnSegmentFile annFile(LocalFileIO fileIO) { + return new PkVectorAnnSegmentFile(fileIO, pathFactory()); + } + + private static DataField vectorField() { + return new DataField(7, "embedding", DataTypes.VECTOR(2, DataTypes.FLOAT())); + } + + private static Options indexOptions() { + Options options = new Options(); + options.setString("test.vector.dimension", "2"); + options.setString("test.vector.metric", "l2"); + return options; } private static DataFileMeta dataFile(String fileName, long rowCount) { @@ -295,7 +168,7 @@ private static DataFileMeta dataFile(String fileName, long rowCount) { 1, Collections.emptyList(), null, - FileSource.APPEND, + FileSource.COMPACT, null, null, null, @@ -321,4 +194,37 @@ public boolean isExternalPath() { } }; } + + private static class ArrayReader implements PkVectorReader { + + private final float[][] vectors; + private int position; + + private ArrayReader(float[][] vectors) { + this.vectors = vectors; + } + + @Override + public int dimension() { + return 2; + } + + @Override + public long rowCount() { + return vectors.length; + } + + @Override + public boolean readNextVector(float[] reuse) { + float[] vector = vectors[position++]; + if (vector == null) { + return false; + } + System.arraycopy(vector, 0, reuse, 0, reuse.length); + return true; + } + + @Override + public void close() throws IOException {} + } } diff --git a/paimon-core/src/test/java/org/apache/paimon/index/pkvector/PkVectorRawSegmentMetaTest.java b/paimon-core/src/test/java/org/apache/paimon/index/pkvector/PkVectorRawSegmentMetaTest.java deleted file mode 100644 index 6f48105fa596..000000000000 --- a/paimon-core/src/test/java/org/apache/paimon/index/pkvector/PkVectorRawSegmentMetaTest.java +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.paimon.index.pkvector; - -import org.junit.jupiter.api.Test; - -import java.util.Arrays; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.assertThatThrownBy; - -/** Tests for {@link PkVectorRawSegmentMeta}. */ -class PkVectorRawSegmentMetaTest { - - @Test - void testRoundTrip() { - PkVectorRawSegmentMeta metadata = - new PkVectorRawSegmentMeta( - "index-definition", new PkVectorSourceFile("data-1", 100)); - - PkVectorRawSegmentMeta restored = PkVectorRawSegmentMeta.deserialize(metadata.serialize()); - - assertThat(restored.indexDefinitionId()).isEqualTo("index-definition"); - assertThat(restored.sourceFile()).isEqualTo(new PkVectorSourceFile("data-1", 100)); - } - - @Test - void testRejectsTrailingBytes() { - PkVectorRawSegmentMeta metadata = - new PkVectorRawSegmentMeta("index", new PkVectorSourceFile("data", 1)); - byte[] bytes = Arrays.copyOf(metadata.serialize(), metadata.serialize().length + 1); - - assertThatThrownBy(() -> PkVectorRawSegmentMeta.deserialize(bytes)) - .hasMessageContaining("Unexpected trailing bytes"); - } -} diff --git a/paimon-core/src/test/java/org/apache/paimon/index/pkvector/RawVectorSidecarTest.java b/paimon-core/src/test/java/org/apache/paimon/index/pkvector/RawVectorSidecarTest.java deleted file mode 100644 index 883e37e51447..000000000000 --- a/paimon-core/src/test/java/org/apache/paimon/index/pkvector/RawVectorSidecarTest.java +++ /dev/null @@ -1,166 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.paimon.index.pkvector; - -import org.apache.paimon.data.GenericArray; -import org.apache.paimon.fs.Path; -import org.apache.paimon.fs.local.LocalFileIO; - -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.io.TempDir; - -import java.util.List; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.assertThatThrownBy; - -/** Tests for raw vector sidecars. */ -class RawVectorSidecarTest { - - @TempDir java.nio.file.Path tempPath; - - @Test - void testRoundTripByRowPosition() throws Exception { - LocalFileIO fileIO = LocalFileIO.create(); - Path path = new Path(tempPath.resolve("raw-vector").toUri()); - try (RawVectorSidecarWriter writer = new RawVectorSidecarWriter(fileIO, path, 3)) { - writer.write(new float[] {1, 2, 3}); - writer.write(null); - writer.write(new GenericArray(new float[] {4, 5, 6})); - } - - try (RawVectorSidecarReader reader = new RawVectorSidecarReader(fileIO, path)) { - assertThat(reader.dimension()).isEqualTo(3); - assertThat(reader.rowCount()).isEqualTo(3); - assertThat(reader.rowPositionForOrdinal(2)).isEqualTo(2); - assertThat(reader.readVector(2)).containsExactly(4, 5, 6); - assertThat(reader.readVector(1)).isNull(); - assertThat(reader.readVector(0)).containsExactly(1, 2, 3); - } - } - - @Test - void testRejectNonFiniteVectorWithoutCorruptingFile() throws Exception { - LocalFileIO fileIO = LocalFileIO.create(); - Path path = new Path(tempPath.resolve("finite-vector").toUri()); - try (RawVectorSidecarWriter writer = new RawVectorSidecarWriter(fileIO, path, 2)) { - assertThatThrownBy(() -> writer.write(new float[] {Float.NaN, 1})) - .isInstanceOf(IllegalArgumentException.class) - .hasMessageContaining("must be finite"); - assertThat(writer.rowCount()).isZero(); - writer.write(new float[] {2, 3}); - } - - try (RawVectorSidecarReader reader = new RawVectorSidecarReader(fileIO, path)) { - assertThat(reader.rowCount()).isEqualTo(1); - assertThat(reader.readVector(0)).containsExactly(2, 3); - } - } - - @Test - void testSequentialReadIntoReusableBuffer() throws Exception { - LocalFileIO fileIO = LocalFileIO.create(); - Path path = new Path(tempPath.resolve("sequential-vector").toUri()); - try (RawVectorSidecarWriter writer = new RawVectorSidecarWriter(fileIO, path, 2)) { - writer.write(new float[] {1, 2}); - writer.write(null); - writer.write(new float[] {3, 4}); - } - - try (RawVectorSidecarReader reader = new RawVectorSidecarReader(fileIO, path)) { - float[] buffer = new float[2]; - reader.rewind(); - assertThat(reader.readNextVector(buffer)).isTrue(); - assertThat(buffer).containsExactly(1, 2); - assertThat(reader.readNextVector(buffer)).isFalse(); - assertThat(reader.readNextVector(buffer)).isTrue(); - assertThat(buffer).containsExactly(3, 4); - assertThatThrownBy(() -> reader.readNextVector(buffer)) - .hasMessageContaining("No raw vector remains"); - } - } - - @Test - void testExactSearchSkipsNullAndDeletedPositions() throws Exception { - LocalFileIO fileIO = LocalFileIO.create(); - Path path = new Path(tempPath.resolve("search-vector").toUri()); - try (RawVectorSidecarWriter writer = new RawVectorSidecarWriter(fileIO, path, 2)) { - writer.write(new float[] {0, 0}); - writer.write(null); - writer.write(new float[] {1, 0}); - writer.write(new float[] {0, 2}); - writer.write(new float[] {3, 3}); - } - - try (RawVectorSidecarReader reader = new RawVectorSidecarReader(fileIO, path)) { - List candidates = - reader.search(new float[] {0, 0}, "l2", 2, position -> position == 0); - - assertThat(candidates) - .extracting(RawVectorSidecarReader.Candidate::rowPosition) - .containsExactly(2L, 3L); - assertThat(candidates) - .extracting(RawVectorSidecarReader.Candidate::distance) - .containsExactly(1.0f, 4.0f); - } - } - - @Test - void testCosineSearch() throws Exception { - LocalFileIO fileIO = LocalFileIO.create(); - Path path = new Path(tempPath.resolve("cosine-vector").toUri()); - try (RawVectorSidecarWriter writer = new RawVectorSidecarWriter(fileIO, path, 2)) { - writer.write(new float[] {1, 0}); - writer.write(new float[] {0, 1}); - writer.write(new float[] {2, 0}); - } - - try (RawVectorSidecarReader reader = new RawVectorSidecarReader(fileIO, path)) { - List candidates = - reader.search(new float[] {1, 0}, "cosine", 3, position -> false); - - assertThat(candidates) - .extracting(RawVectorSidecarReader.Candidate::rowPosition) - .containsExactly(0L, 2L, 1L); - assertThat(candidates) - .extracting(RawVectorSidecarReader.Candidate::distance) - .containsExactly(0.0f, 0.0f, 1.0f); - } - } - - @Test - void testInnerProductSearch() throws Exception { - LocalFileIO fileIO = LocalFileIO.create(); - Path path = new Path(tempPath.resolve("inner-product-vector").toUri()); - try (RawVectorSidecarWriter writer = new RawVectorSidecarWriter(fileIO, path, 2)) { - writer.write(new float[] {1, 0}); - writer.write(new float[] {0, 1}); - writer.write(new float[] {2, 0}); - } - - try (RawVectorSidecarReader reader = new RawVectorSidecarReader(fileIO, path)) { - List candidates = - reader.search(new float[] {1, 0}, "inner_product", 3, position -> false); - - assertThat(candidates) - .extracting(RawVectorSidecarReader.Candidate::rowPosition) - .containsExactly(2L, 0L, 1L); - } - } -} From 6cdcb3cb7235be11d1b272f0cb7499224864f3ed Mon Sep 17 00:00:00 2001 From: JingsongLi Date: Sat, 11 Jul 2026 19:49:08 +0800 Subject: [PATCH 13/19] [core] Remove redundant ANN ordinal layout --- .../pkvector/PkVectorAnnSegmentFile.java | 4 +-- .../pkvector/PkVectorAnnSegmentMeta.java | 29 +------------------ .../pkvector/PkVectorAnnSegmentSearcher.java | 12 -------- .../pkvector/PkVectorAnnSegmentFileTest.java | 6 ---- .../pkvector/PkVectorAnnSegmentMetaTest.java | 4 --- 5 files changed, 2 insertions(+), 53 deletions(-) diff --git a/paimon-core/src/main/java/org/apache/paimon/index/pkvector/PkVectorAnnSegmentFile.java b/paimon-core/src/main/java/org/apache/paimon/index/pkvector/PkVectorAnnSegmentFile.java index 926256e4738b..8e492a448cf0 100644 --- a/paimon-core/src/main/java/org/apache/paimon/index/pkvector/PkVectorAnnSegmentFile.java +++ b/paimon-core/src/main/java/org/apache/paimon/index/pkvector/PkVectorAnnSegmentFile.java @@ -60,7 +60,6 @@ public PkVectorAnnSegmentFile(FileIO fileIO, IndexPathFactory pathFactory) { public IndexFileMeta build( List sources, - PkVectorAnnSegmentMeta.OrdinalLayout ordinalLayout, DataField vectorField, Options indexOptions, String indexDefinitionId, @@ -151,8 +150,7 @@ public IndexFileMeta build( Path payloadPath = fileWriter.path(result.fileName()); byte[] payloadMetadata = result.meta() == null ? new byte[0] : result.meta(); PkVectorAnnSegmentMeta metadata = - new PkVectorAnnSegmentMeta( - indexDefinitionId, sourceFiles, ordinalLayout, payloadMetadata); + new PkVectorAnnSegmentMeta(indexDefinitionId, sourceFiles, payloadMetadata); IndexFileMeta segment = new IndexFileMeta( PK_VECTOR_ANN, diff --git a/paimon-core/src/main/java/org/apache/paimon/index/pkvector/PkVectorAnnSegmentMeta.java b/paimon-core/src/main/java/org/apache/paimon/index/pkvector/PkVectorAnnSegmentMeta.java index df8eaa3e83d2..6a3f36317d7c 100644 --- a/paimon-core/src/main/java/org/apache/paimon/index/pkvector/PkVectorAnnSegmentMeta.java +++ b/paimon-core/src/main/java/org/apache/paimon/index/pkvector/PkVectorAnnSegmentMeta.java @@ -37,17 +37,14 @@ public final class PkVectorAnnSegmentMeta { private final String indexDefinitionId; private final List sourceFiles; - private final OrdinalLayout ordinalLayout; private final byte[] payloadMetadata; public PkVectorAnnSegmentMeta( String indexDefinitionId, List sourceFiles, - OrdinalLayout ordinalLayout, byte[] payloadMetadata) { this.indexDefinitionId = Objects.requireNonNull(indexDefinitionId); this.sourceFiles = Collections.unmodifiableList(new ArrayList<>(sourceFiles)); - this.ordinalLayout = Objects.requireNonNull(ordinalLayout); this.payloadMetadata = Arrays.copyOf(payloadMetadata, payloadMetadata.length); checkArgument(!this.sourceFiles.isEmpty(), "An ANN segment must reference source files."); } @@ -60,10 +57,6 @@ public List sourceFiles() { return sourceFiles; } - public OrdinalLayout ordinalLayout() { - return ordinalLayout; - } - public byte[] payloadMetadata() { return Arrays.copyOf(payloadMetadata, payloadMetadata.length); } @@ -78,7 +71,6 @@ public byte[] serialize() { output.writeUTF(sourceFile.fileName()); output.writeLong(sourceFile.rowCount()); } - output.writeByte(ordinalLayout.ordinal()); output.writeInt(payloadMetadata.length); output.write(payloadMetadata); return output.getCopyOfBuffer(); @@ -100,8 +92,6 @@ public static PkVectorAnnSegmentMeta deserialize(byte[] bytes) { for (int i = 0; i < sourceFileCount; i++) { sourceFiles.add(new PkVectorSourceFile(input.readUTF(), input.readLong())); } - OrdinalLayout ordinalLayout = - enumValue(OrdinalLayout.values(), input.readByte(), "ordinal layout"); int payloadMetadataLength = input.readInt(); checkArgument( payloadMetadataLength >= 0, "Payload metadata length must not be negative."); @@ -110,27 +100,10 @@ public static PkVectorAnnSegmentMeta deserialize(byte[] bytes) { checkArgument( input.available() == 0, "Unexpected trailing bytes in ANN vector segment metadata."); - return new PkVectorAnnSegmentMeta( - indexDefinitionId, sourceFiles, ordinalLayout, payloadMetadata); + return new PkVectorAnnSegmentMeta(indexDefinitionId, sourceFiles, payloadMetadata); } catch (IOException e) { throw new IllegalArgumentException( "Failed to deserialize ANN vector segment metadata.", e); } } - - private static T enumValue(T[] values, byte ordinal, String field) { - int index = ordinal; - checkArgument( - index >= 0 && index < values.length, - "Unknown ANN vector segment %s: %s.", - field, - ordinal); - return values[index]; - } - - /** Mapping from an ANN-local ordinal to a physical data-file position. */ - public enum OrdinalLayout { - ROW_POSITION, - FILE_POSITION - } } diff --git a/paimon-core/src/main/java/org/apache/paimon/index/pkvector/PkVectorAnnSegmentSearcher.java b/paimon-core/src/main/java/org/apache/paimon/index/pkvector/PkVectorAnnSegmentSearcher.java index 9196c21de3d2..3b91b86f9985 100644 --- a/paimon-core/src/main/java/org/apache/paimon/index/pkvector/PkVectorAnnSegmentSearcher.java +++ b/paimon-core/src/main/java/org/apache/paimon/index/pkvector/PkVectorAnnSegmentSearcher.java @@ -45,8 +45,6 @@ import java.util.Optional; import java.util.concurrent.ExecutorService; -import static org.apache.paimon.index.pkvector.PkVectorAnnSegmentMeta.OrdinalLayout.FILE_POSITION; -import static org.apache.paimon.index.pkvector.PkVectorAnnSegmentMeta.OrdinalLayout.ROW_POSITION; import static org.apache.paimon.utils.Preconditions.checkArgument; /** Searches one ANN payload and maps its segment-local ids back to source row positions. */ @@ -111,16 +109,6 @@ public List search( PkVectorAnnSegmentFile.PK_VECTOR_ANN.equals(segment.indexType()), "Vector segment %s is not an ANN payload.", segment.fileName()); - checkArgument( - metadata.ordinalLayout() == ROW_POSITION - || metadata.ordinalLayout() == FILE_POSITION, - "ANN segment %s has unsupported ordinal layout %s.", - segment.fileName(), - metadata.ordinalLayout()); - checkArgument( - metadata.ordinalLayout() != ROW_POSITION || metadata.sourceFiles().size() == 1, - "Row-position ANN segment %s must reference exactly one source file.", - segment.fileName()); GlobalIndexer indexer = GlobalIndexer.create(algorithm, vectorField, indexOptions); checkArgument( indexer instanceof VectorGlobalIndexer, diff --git a/paimon-core/src/test/java/org/apache/paimon/index/pkvector/PkVectorAnnSegmentFileTest.java b/paimon-core/src/test/java/org/apache/paimon/index/pkvector/PkVectorAnnSegmentFileTest.java index 106d24716f1d..54e50a661610 100644 --- a/paimon-core/src/test/java/org/apache/paimon/index/pkvector/PkVectorAnnSegmentFileTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/index/pkvector/PkVectorAnnSegmentFileTest.java @@ -43,8 +43,6 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; -import static org.apache.paimon.index.pkvector.PkVectorAnnSegmentMeta.OrdinalLayout.FILE_POSITION; -import static org.apache.paimon.index.pkvector.PkVectorAnnSegmentMeta.OrdinalLayout.ROW_POSITION; import static org.assertj.core.api.Assertions.assertThat; /** Tests ANN construction from generic vector readers. */ @@ -64,7 +62,6 @@ void testBuildSkipsNullAndExcludedPhysicalRows() throws Exception { new ArrayReader( new float[][] {{0, 0}, null, {2, 0}}), position -> position == 0)), - ROW_POSITION, vectorField(), indexOptions(), "definition", @@ -75,7 +72,6 @@ void testBuildSkipsNullAndExcludedPhysicalRows() throws Exception { assertThat(segment.rowCount()).isEqualTo(1); PkVectorAnnSegmentMeta metadata = PkVectorAnnSegmentMeta.deserialize(segment.globalIndexMeta().indexMeta()); - assertThat(metadata.ordinalLayout()).isEqualTo(ROW_POSITION); assertThat(metadata.sourceFiles()) .extracting(PkVectorSourceFile::fileName) .containsExactly("data-1"); @@ -94,7 +90,6 @@ void testBuildsAndSearchesMultiSourceSegment() throws Exception { new PkVectorAnnSegmentFile.Source( dataFile("data-2", 2), new ArrayReader(new float[][] {{0, 0}, {2, 0}}))), - FILE_POSITION, vectorField(), indexOptions(), "definition", @@ -131,7 +126,6 @@ void testBuildsAndSearchesMultiSourceSegment() throws Exception { executor.shutdownNow(); } - assertThat(metadata.ordinalLayout()).isEqualTo(FILE_POSITION); assertThat(candidates) .extracting( PkVectorAnnSegmentSearcher.Candidate::dataFileName, diff --git a/paimon-core/src/test/java/org/apache/paimon/index/pkvector/PkVectorAnnSegmentMetaTest.java b/paimon-core/src/test/java/org/apache/paimon/index/pkvector/PkVectorAnnSegmentMetaTest.java index 36051763ccbe..5c683cf0daeb 100644 --- a/paimon-core/src/test/java/org/apache/paimon/index/pkvector/PkVectorAnnSegmentMetaTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/index/pkvector/PkVectorAnnSegmentMetaTest.java @@ -38,15 +38,12 @@ void testRoundTrip() { Arrays.asList( new PkVectorSourceFile("data-1", 100), new PkVectorSourceFile("data-2", 50)), - PkVectorAnnSegmentMeta.OrdinalLayout.FILE_POSITION, new byte[] {1, 2, 3}); PkVectorAnnSegmentMeta restored = PkVectorAnnSegmentMeta.deserialize(metadata.serialize()); assertThat(restored.indexDefinitionId()).isEqualTo("index-definition"); assertThat(restored.sourceFiles()).isEqualTo(metadata.sourceFiles()); - assertThat(restored.ordinalLayout()) - .isEqualTo(PkVectorAnnSegmentMeta.OrdinalLayout.FILE_POSITION); assertThat(restored.payloadMetadata()).containsExactly(1, 2, 3); } @@ -58,7 +55,6 @@ void testRejectsTruncatedPayloadMetadata() throws Exception { output.writeInt(1); output.writeUTF("data-1"); output.writeLong(10); - output.writeByte(PkVectorAnnSegmentMeta.OrdinalLayout.FILE_POSITION.ordinal()); output.writeInt(1); assertThatThrownBy(() -> PkVectorAnnSegmentMeta.deserialize(output.getCopyOfBuffer())) From b3d4fdbcdf83d185b8b479630e79b99d54e64e72 Mon Sep 17 00:00:00 2001 From: JingsongLi Date: Sat, 11 Jul 2026 20:02:40 +0800 Subject: [PATCH 14/19] [core] Persist primary-key vector index type --- .../pkvector/PkVectorAnnSegmentFile.java | 11 ++- .../pkvector/PkVectorAnnSegmentMeta.java | 18 +++-- .../pkvector/PkVectorAnnSegmentSearcher.java | 8 +-- .../PrimaryKeyVectorIndexOptions.java | 61 +---------------- .../pkvector/PkVectorAnnSegmentFileTest.java | 12 +--- .../pkvector/PkVectorAnnSegmentMetaTest.java | 4 +- .../PrimaryKeyVectorIndexOptionsTest.java | 67 ------------------- 7 files changed, 22 insertions(+), 159 deletions(-) diff --git a/paimon-core/src/main/java/org/apache/paimon/index/pkvector/PkVectorAnnSegmentFile.java b/paimon-core/src/main/java/org/apache/paimon/index/pkvector/PkVectorAnnSegmentFile.java index 8e492a448cf0..6f2a0569a70a 100644 --- a/paimon-core/src/main/java/org/apache/paimon/index/pkvector/PkVectorAnnSegmentFile.java +++ b/paimon-core/src/main/java/org/apache/paimon/index/pkvector/PkVectorAnnSegmentFile.java @@ -62,9 +62,8 @@ public IndexFileMeta build( List sources, DataField vectorField, Options indexOptions, - String indexDefinitionId, String metric, - String algorithm) + String indexType) throws IOException { checkArgument(!sources.isEmpty(), "An ANN segment must reference source files."); long totalRowCount = 0; @@ -74,11 +73,11 @@ public IndexFileMeta build( sourceFiles.add(source.sourceFile); } - GlobalIndexer indexer = GlobalIndexer.create(algorithm, vectorField, indexOptions); + GlobalIndexer indexer = GlobalIndexer.create(indexType, vectorField, indexOptions); checkArgument( indexer instanceof VectorGlobalIndexer, "Index algorithm %s does not implement VectorGlobalIndexer.", - algorithm); + indexType); String indexerMetric = normalizeMetric(((VectorGlobalIndexer) indexer).metric()); checkArgument( normalizeMetric(metric).equals(indexerMetric), @@ -94,7 +93,7 @@ public IndexFileMeta build( checkArgument( writer instanceof GlobalIndexSingleColumnWriter, "Index algorithm %s does not create a single-column writer.", - algorithm); + indexType); GlobalIndexSingleColumnWriter singleColumnWriter = (GlobalIndexSingleColumnWriter) writer; long liveRowCount = 0; @@ -150,7 +149,7 @@ public IndexFileMeta build( Path payloadPath = fileWriter.path(result.fileName()); byte[] payloadMetadata = result.meta() == null ? new byte[0] : result.meta(); PkVectorAnnSegmentMeta metadata = - new PkVectorAnnSegmentMeta(indexDefinitionId, sourceFiles, payloadMetadata); + new PkVectorAnnSegmentMeta(indexType, sourceFiles, payloadMetadata); IndexFileMeta segment = new IndexFileMeta( PK_VECTOR_ANN, diff --git a/paimon-core/src/main/java/org/apache/paimon/index/pkvector/PkVectorAnnSegmentMeta.java b/paimon-core/src/main/java/org/apache/paimon/index/pkvector/PkVectorAnnSegmentMeta.java index 6a3f36317d7c..368ca2575a5a 100644 --- a/paimon-core/src/main/java/org/apache/paimon/index/pkvector/PkVectorAnnSegmentMeta.java +++ b/paimon-core/src/main/java/org/apache/paimon/index/pkvector/PkVectorAnnSegmentMeta.java @@ -35,22 +35,20 @@ public final class PkVectorAnnSegmentMeta { private static final int VERSION = 1; - private final String indexDefinitionId; + private final String indexType; private final List sourceFiles; private final byte[] payloadMetadata; public PkVectorAnnSegmentMeta( - String indexDefinitionId, - List sourceFiles, - byte[] payloadMetadata) { - this.indexDefinitionId = Objects.requireNonNull(indexDefinitionId); + String indexType, List sourceFiles, byte[] payloadMetadata) { + this.indexType = Objects.requireNonNull(indexType); this.sourceFiles = Collections.unmodifiableList(new ArrayList<>(sourceFiles)); this.payloadMetadata = Arrays.copyOf(payloadMetadata, payloadMetadata.length); checkArgument(!this.sourceFiles.isEmpty(), "An ANN segment must reference source files."); } - public String indexDefinitionId() { - return indexDefinitionId; + public String indexType() { + return indexType; } public List sourceFiles() { @@ -65,7 +63,7 @@ public byte[] serialize() { try { DataOutputSerializer output = new DataOutputSerializer(128); output.writeInt(VERSION); - output.writeUTF(indexDefinitionId); + output.writeUTF(indexType); output.writeInt(sourceFiles.size()); for (PkVectorSourceFile sourceFile : sourceFiles) { output.writeUTF(sourceFile.fileName()); @@ -85,7 +83,7 @@ public static PkVectorAnnSegmentMeta deserialize(byte[] bytes) { int version = input.readInt(); checkArgument( version == VERSION, "Unsupported ANN vector segment version: %s.", version); - String indexDefinitionId = input.readUTF(); + String indexType = input.readUTF(); int sourceFileCount = input.readInt(); checkArgument(sourceFileCount > 0, "An ANN segment must reference source files."); List sourceFiles = new ArrayList<>(sourceFileCount); @@ -100,7 +98,7 @@ public static PkVectorAnnSegmentMeta deserialize(byte[] bytes) { checkArgument( input.available() == 0, "Unexpected trailing bytes in ANN vector segment metadata."); - return new PkVectorAnnSegmentMeta(indexDefinitionId, sourceFiles, payloadMetadata); + return new PkVectorAnnSegmentMeta(indexType, sourceFiles, payloadMetadata); } catch (IOException e) { throw new IllegalArgumentException( "Failed to deserialize ANN vector segment metadata.", e); diff --git a/paimon-core/src/main/java/org/apache/paimon/index/pkvector/PkVectorAnnSegmentSearcher.java b/paimon-core/src/main/java/org/apache/paimon/index/pkvector/PkVectorAnnSegmentSearcher.java index 3b91b86f9985..a8f69cf7c26e 100644 --- a/paimon-core/src/main/java/org/apache/paimon/index/pkvector/PkVectorAnnSegmentSearcher.java +++ b/paimon-core/src/main/java/org/apache/paimon/index/pkvector/PkVectorAnnSegmentSearcher.java @@ -59,7 +59,6 @@ public class PkVectorAnnSegmentSearcher { private final PkVectorAnnSegmentFile annSegmentFile; private final DataField vectorField; private final Options indexOptions; - private final String algorithm; private final String metric; private final ExecutorService executor; @@ -68,14 +67,12 @@ public PkVectorAnnSegmentSearcher( PkVectorAnnSegmentFile annSegmentFile, DataField vectorField, Options indexOptions, - String algorithm, String metric, ExecutorService executor) { this.fileIO = fileIO; this.annSegmentFile = annSegmentFile; this.vectorField = vectorField; this.indexOptions = indexOptions; - this.algorithm = algorithm; this.metric = normalizeMetric(metric); this.executor = executor; } @@ -109,11 +106,12 @@ public List search( PkVectorAnnSegmentFile.PK_VECTOR_ANN.equals(segment.indexType()), "Vector segment %s is not an ANN payload.", segment.fileName()); - GlobalIndexer indexer = GlobalIndexer.create(algorithm, vectorField, indexOptions); + GlobalIndexer indexer = + GlobalIndexer.create(metadata.indexType(), vectorField, indexOptions); checkArgument( indexer instanceof VectorGlobalIndexer, "Index algorithm %s does not implement VectorGlobalIndexer.", - algorithm); + metadata.indexType()); String readerMetric = normalizeMetric(((VectorGlobalIndexer) indexer).metric()); checkArgument( metric.equals(readerMetric), diff --git a/paimon-core/src/main/java/org/apache/paimon/index/pkvector/PrimaryKeyVectorIndexOptions.java b/paimon-core/src/main/java/org/apache/paimon/index/pkvector/PrimaryKeyVectorIndexOptions.java index d27a1588b593..46ea22ad6ed1 100644 --- a/paimon-core/src/main/java/org/apache/paimon/index/pkvector/PrimaryKeyVectorIndexOptions.java +++ b/paimon-core/src/main/java/org/apache/paimon/index/pkvector/PrimaryKeyVectorIndexOptions.java @@ -21,11 +21,7 @@ import org.apache.paimon.CoreOptions; import org.apache.paimon.options.Options; import org.apache.paimon.utils.JsonSerdeUtil; -import org.apache.paimon.utils.StringUtils; -import java.nio.charset.StandardCharsets; -import java.security.MessageDigest; -import java.security.NoSuchAlgorithmException; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -33,7 +29,7 @@ import static org.apache.paimon.utils.Preconditions.checkArgument; -/** Resolves and fingerprints algorithm options for the primary-key vector index. */ +/** Resolves algorithm options for the primary-key vector index. */ public final class PrimaryKeyVectorIndexOptions { private PrimaryKeyVectorIndexOptions() {} @@ -50,35 +46,6 @@ public static Options resolve(CoreOptions coreOptions, String field) { return resolved; } - public static byte[] hash(CoreOptions coreOptions) { - return hash(coreOptions, singleColumn(coreOptions)); - } - - public static byte[] hash(CoreOptions coreOptions, String field) { - return sha256(JsonSerdeUtil.toJson(fingerprintOptions(coreOptions, field))); - } - - public static String definitionId( - int vectorFieldId, String vectorTypeFingerprint, CoreOptions coreOptions) { - return definitionId( - vectorFieldId, vectorTypeFingerprint, coreOptions, singleColumn(coreOptions)); - } - - public static String definitionId( - int vectorFieldId, - String vectorTypeFingerprint, - CoreOptions coreOptions, - String field) { - checkArgument( - vectorTypeFingerprint != null && !vectorTypeFingerprint.trim().isEmpty(), - "Vector type fingerprint must not be empty."); - TreeMap definition = new TreeMap<>(); - definition.put("field-id", Integer.toString(vectorFieldId)); - definition.put("field-type", vectorTypeFingerprint); - definition.putAll(fingerprintOptions(coreOptions, field)); - return StringUtils.byteToHexString(sha256(JsonSerdeUtil.toJson(definition))); - } - public static String singleColumn(CoreOptions coreOptions) { List columns = coreOptions.primaryKeyVectorIndexColumns(); checkArgument( @@ -88,15 +55,6 @@ public static String singleColumn(CoreOptions coreOptions) { return columns.get(0); } - private static byte[] sha256(String value) { - try { - return MessageDigest.getInstance("SHA-256") - .digest(value.getBytes(StandardCharsets.UTF_8)); - } catch (NoSuchAlgorithmException e) { - throw new IllegalStateException("SHA-256 is not available.", e); - } - } - private static Map algorithmOptions(CoreOptions coreOptions, String field) { String indexTypeKey = "fields." + field + ".pk-vector.index.type"; String indexOptionsKey = "fields." + field + ".pk-vector.index.options"; @@ -151,21 +109,4 @@ private static Map algorithmOptions(CoreOptions coreOptions, Str options.put(algorithmPrefix + "metric", coreOptions.primaryKeyVectorDistanceMetric(field)); return options; } - - private static Map fingerprintOptions(CoreOptions coreOptions, String field) { - String fieldPrefix = "fields." + field + "."; - TreeMap fingerprint = new TreeMap<>(); - Map options = algorithmOptions(coreOptions, field); - for (Map.Entry entry : options.entrySet()) { - if (!entry.getKey().startsWith(fieldPrefix)) { - fingerprint.put(entry.getKey(), entry.getValue()); - } - } - for (Map.Entry entry : options.entrySet()) { - if (entry.getKey().startsWith(fieldPrefix)) { - fingerprint.put(entry.getKey().substring(fieldPrefix.length()), entry.getValue()); - } - } - return fingerprint; - } } diff --git a/paimon-core/src/test/java/org/apache/paimon/index/pkvector/PkVectorAnnSegmentFileTest.java b/paimon-core/src/test/java/org/apache/paimon/index/pkvector/PkVectorAnnSegmentFileTest.java index 54e50a661610..97ed4d379e86 100644 --- a/paimon-core/src/test/java/org/apache/paimon/index/pkvector/PkVectorAnnSegmentFileTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/index/pkvector/PkVectorAnnSegmentFileTest.java @@ -64,7 +64,6 @@ void testBuildSkipsNullAndExcludedPhysicalRows() throws Exception { position -> position == 0)), vectorField(), indexOptions(), - "definition", "l2", "test-vector-ann"); @@ -72,6 +71,7 @@ void testBuildSkipsNullAndExcludedPhysicalRows() throws Exception { assertThat(segment.rowCount()).isEqualTo(1); PkVectorAnnSegmentMeta metadata = PkVectorAnnSegmentMeta.deserialize(segment.globalIndexMeta().indexMeta()); + assertThat(metadata.indexType()).isEqualTo("test-vector-ann"); assertThat(metadata.sourceFiles()) .extracting(PkVectorSourceFile::fileName) .containsExactly("data-1"); @@ -92,11 +92,11 @@ void testBuildsAndSearchesMultiSourceSegment() throws Exception { new ArrayReader(new float[][] {{0, 0}, {2, 0}}))), vectorField(), indexOptions(), - "definition", "l2", "test-vector-ann"); PkVectorAnnSegmentMeta metadata = PkVectorAnnSegmentMeta.deserialize(segment.globalIndexMeta().indexMeta()); + assertThat(metadata.indexType()).isEqualTo("test-vector-ann"); BitmapDeletionVector data2Deletes = new BitmapDeletionVector(); data2Deletes.delete(0); Map deletionVectors = @@ -108,13 +108,7 @@ void testBuildsAndSearchesMultiSourceSegment() throws Exception { try { candidates = new PkVectorAnnSegmentSearcher( - fileIO, - annFile, - vectorField(), - indexOptions(), - "test-vector-ann", - "l2", - executor) + fileIO, annFile, vectorField(), indexOptions(), "l2", executor) .search( segment, metadata, diff --git a/paimon-core/src/test/java/org/apache/paimon/index/pkvector/PkVectorAnnSegmentMetaTest.java b/paimon-core/src/test/java/org/apache/paimon/index/pkvector/PkVectorAnnSegmentMetaTest.java index 5c683cf0daeb..8284af0f2de6 100644 --- a/paimon-core/src/test/java/org/apache/paimon/index/pkvector/PkVectorAnnSegmentMetaTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/index/pkvector/PkVectorAnnSegmentMetaTest.java @@ -34,7 +34,7 @@ class PkVectorAnnSegmentMetaTest { void testRoundTrip() { PkVectorAnnSegmentMeta metadata = new PkVectorAnnSegmentMeta( - "index-definition", + "test-vector-ann", Arrays.asList( new PkVectorSourceFile("data-1", 100), new PkVectorSourceFile("data-2", 50)), @@ -42,7 +42,7 @@ void testRoundTrip() { PkVectorAnnSegmentMeta restored = PkVectorAnnSegmentMeta.deserialize(metadata.serialize()); - assertThat(restored.indexDefinitionId()).isEqualTo("index-definition"); + assertThat(restored.indexType()).isEqualTo("test-vector-ann"); assertThat(restored.sourceFiles()).isEqualTo(metadata.sourceFiles()); assertThat(restored.payloadMetadata()).containsExactly(1, 2, 3); } diff --git a/paimon-core/src/test/java/org/apache/paimon/index/pkvector/PrimaryKeyVectorIndexOptionsTest.java b/paimon-core/src/test/java/org/apache/paimon/index/pkvector/PrimaryKeyVectorIndexOptionsTest.java index 5b015f188ee1..90084558fbcb 100644 --- a/paimon-core/src/test/java/org/apache/paimon/index/pkvector/PrimaryKeyVectorIndexOptionsTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/index/pkvector/PrimaryKeyVectorIndexOptionsTest.java @@ -126,73 +126,6 @@ void testResolvesShortAndQualifiedAlgorithmOptions() { assertThat(resolved.get("ivf-pq.metric")).isEqualTo("l2"); } - @Test - void testHashIsCanonicalAcrossJsonPropertyOrder() { - assertThat(PrimaryKeyVectorIndexOptions.hash(coreOptions("{\"nlist\":64,\"pq.m\":8}"))) - .containsExactly( - PrimaryKeyVectorIndexOptions.hash( - coreOptions("{\"pq.m\":\"8\",\"nlist\":\"64\"}"))); - } - - @Test - void testHashIncludesEffectiveTopLevelAlgorithmOptions() { - assertThat(PrimaryKeyVectorIndexOptions.hash(coreOptions(null, "ivf-pq.nlist", "64"))) - .isNotEqualTo( - PrimaryKeyVectorIndexOptions.hash(coreOptions(null, "ivf-pq.nlist", "65"))); - } - - @Test - void testDefinitionIdIsStableAndDefinitionSensitive() { - CoreOptions first = coreOptions("{\"nlist\":64,\"pq.m\":8}"); - CoreOptions reordered = coreOptions("{\"pq.m\":8,\"nlist\":64}"); - String definitionId = - PrimaryKeyVectorIndexOptions.definitionId(7, "VECTOR", first); - - assertThat(PrimaryKeyVectorIndexOptions.definitionId(7, "VECTOR", reordered)) - .isEqualTo(definitionId); - assertThat(PrimaryKeyVectorIndexOptions.definitionId(8, "VECTOR", first)) - .isNotEqualTo(definitionId); - assertThat(PrimaryKeyVectorIndexOptions.definitionId(7, "VECTOR", first)) - .isNotEqualTo(definitionId); - assertThat( - PrimaryKeyVectorIndexOptions.definitionId( - 7, "VECTOR", coreOptions("{\"nlist\":65,\"pq.m\":8}"))) - .isNotEqualTo(definitionId); - } - - @Test - void testDefinitionIdExcludesOperationalThresholds() { - CoreOptions first = coreOptions("{\"nlist\":64}"); - first.toConfiguration().setString("fields.embedding.pk-vector.ann.min-rows", "10000"); - CoreOptions second = coreOptions("{\"nlist\":64}"); - second.toConfiguration().setString("fields.embedding.pk-vector.ann.min-rows", "20000"); - - assertThat(PrimaryKeyVectorIndexOptions.definitionId(7, "VECTOR", first)) - .isEqualTo( - PrimaryKeyVectorIndexOptions.definitionId(7, "VECTOR", second)); - } - - @Test - void testDefinitionIdIgnoresShadowedTableDefault() { - Map firstOptions = new HashMap<>(); - firstOptions.put(CoreOptions.PK_VECTOR_INDEX_COLUMNS.key(), "embedding"); - firstOptions.put("fields.embedding.pk-vector.index.type", "ivf-pq"); - firstOptions.put("ivf-pq.nlist", "64"); - firstOptions.put("fields.embedding.ivf-pq.nlist", "128"); - Map changedDefault = new HashMap<>(firstOptions); - changedDefault.put("ivf-pq.nlist", "96"); - - assertThat( - PrimaryKeyVectorIndexOptions.definitionId( - 7, "VECTOR", new CoreOptions(firstOptions), "embedding")) - .isEqualTo( - PrimaryKeyVectorIndexOptions.definitionId( - 7, - "VECTOR", - new CoreOptions(changedDefault), - "embedding")); - } - @Test void testRejectsNonObjectOptions() { assertThatThrownBy(() -> PrimaryKeyVectorIndexOptions.resolve(coreOptions("[1,2]"))) From 6fafddb00c2eef1e3536cd5098ed4490bcc0f85c Mon Sep 17 00:00:00 2001 From: JingsongLi Date: Sat, 11 Jul 2026 20:34:37 +0800 Subject: [PATCH 15/19] [core] Separate global index source metadata --- .../apache/paimon/index/GlobalIndexMeta.java | 22 +++++++- .../paimon/index/IndexFileMetaSerializer.java | 16 +++++- .../pkvector/PkVectorAnnSegmentFile.java | 13 +++-- .../pkvector/PkVectorAnnSegmentSearcher.java | 28 +++++----- ...gmentMeta.java => PkVectorSourceMeta.java} | 56 +++++++------------ .../IndexManifestEntrySerializer.java | 16 +++++- .../index/IndexFileMetaSerializerTest.java | 23 ++++++++ .../pkvector/PkVectorAnnSegmentFileTest.java | 14 ++--- ...aTest.java => PkVectorSourceMetaTest.java} | 27 ++++----- .../IndexManifestEntrySerializerTest.java | 38 +++++++++++++ 10 files changed, 165 insertions(+), 88 deletions(-) rename paimon-core/src/main/java/org/apache/paimon/index/pkvector/{PkVectorAnnSegmentMeta.java => PkVectorSourceMeta.java} (55%) rename paimon-core/src/test/java/org/apache/paimon/index/pkvector/{PkVectorAnnSegmentMetaTest.java => PkVectorSourceMetaTest.java} (64%) diff --git a/paimon-core/src/main/java/org/apache/paimon/index/GlobalIndexMeta.java b/paimon-core/src/main/java/org/apache/paimon/index/GlobalIndexMeta.java index 83a6224f3e06..727a6536e3bc 100644 --- a/paimon-core/src/main/java/org/apache/paimon/index/GlobalIndexMeta.java +++ b/paimon-core/src/main/java/org/apache/paimon/index/GlobalIndexMeta.java @@ -43,13 +43,15 @@ public class GlobalIndexMeta { new DataField(2, "_INDEX_FIELD_ID", new IntType(false)), new DataField( 3, "_EXTRA_FIELD_IDS", DataTypes.ARRAY(new IntType(false))), - new DataField(4, "_INDEX_META", DataTypes.BYTES()))); + new DataField(4, "_INDEX_META", DataTypes.BYTES()), + new DataField(5, "_SOURCE_META", DataTypes.BYTES()))); private final long rowRangeStart; private final long rowRangeEnd; private final int indexFieldId; @Nullable private final int[] extraFieldIds; @Nullable private final byte[] indexMeta; + @Nullable private final byte[] sourceMeta; public GlobalIndexMeta( long rowRangeStart, @@ -57,11 +59,22 @@ public GlobalIndexMeta( int indexFieldId, @Nullable int[] extraFieldIds, @Nullable byte[] indexMeta) { + this(rowRangeStart, rowRangeEnd, indexFieldId, extraFieldIds, indexMeta, null); + } + + public GlobalIndexMeta( + long rowRangeStart, + long rowRangeEnd, + int indexFieldId, + @Nullable int[] extraFieldIds, + @Nullable byte[] indexMeta, + @Nullable byte[] sourceMeta) { this.rowRangeStart = rowRangeStart; this.rowRangeEnd = rowRangeEnd; this.indexFieldId = indexFieldId; this.extraFieldIds = extraFieldIds; this.indexMeta = indexMeta; + this.sourceMeta = sourceMeta; } public long rowRangeStart() { @@ -85,11 +98,18 @@ public int[] extraFieldIds() { return extraFieldIds; } + /** Metadata produced and consumed by the global-index implementation. */ @Nullable public byte[] indexMeta() { return indexMeta; } + /** Metadata describing how index row ids map to their source data. */ + @Nullable + public byte[] sourceMeta() { + return sourceMeta; + } + /** All indexed field ids in order: the primary {@link #indexFieldId} followed by the rest. */ public List getIndexedFieldIds() { List ids = new ArrayList<>(); diff --git a/paimon-core/src/main/java/org/apache/paimon/index/IndexFileMetaSerializer.java b/paimon-core/src/main/java/org/apache/paimon/index/IndexFileMetaSerializer.java index 6d98e61248bb..a2acfd0ffcf6 100644 --- a/paimon-core/src/main/java/org/apache/paimon/index/IndexFileMetaSerializer.java +++ b/paimon-core/src/main/java/org/apache/paimon/index/IndexFileMetaSerializer.java @@ -50,7 +50,8 @@ public InternalRow toRow(IndexFileMeta record) { globalIndexMeta.extraFieldIds() == null ? null : new GenericArray(globalIndexMeta.extraFieldIds()), - globalIndexMeta.indexMeta()); + globalIndexMeta.indexMeta(), + globalIndexMeta.sourceMeta()); return GenericRow.of( fromString(record.indexType()), fromString(record.fileName()), @@ -65,16 +66,25 @@ public InternalRow toRow(IndexFileMeta record) { public IndexFileMeta fromRow(InternalRow row) { GlobalIndexMeta globalIndexMeta = null; if (!row.isNullAt(6)) { - InternalRow globalIndexRow = row.getRow(6, 5); + InternalRow globalIndexRow = row.getRow(6, 6); Long rowRangeStart = globalIndexRow.getLong(0); Long rowRangeEnd = globalIndexRow.getLong(1); Integer indexFieldId = globalIndexRow.getInt(2); int[] extralFields = globalIndexRow.isNullAt(3) ? null : globalIndexRow.getArray(3).toIntArray(); byte[] indexMeta = globalIndexRow.isNullAt(4) ? null : globalIndexRow.getBinary(4); + byte[] sourceMeta = + globalIndexRow.getFieldCount() <= 5 || globalIndexRow.isNullAt(5) + ? null + : globalIndexRow.getBinary(5); globalIndexMeta = new GlobalIndexMeta( - rowRangeStart, rowRangeEnd, indexFieldId, extralFields, indexMeta); + rowRangeStart, + rowRangeEnd, + indexFieldId, + extralFields, + indexMeta, + sourceMeta); } return new IndexFileMeta( row.getString(0).toString(), diff --git a/paimon-core/src/main/java/org/apache/paimon/index/pkvector/PkVectorAnnSegmentFile.java b/paimon-core/src/main/java/org/apache/paimon/index/pkvector/PkVectorAnnSegmentFile.java index 6f2a0569a70a..aada72eb3e8b 100644 --- a/paimon-core/src/main/java/org/apache/paimon/index/pkvector/PkVectorAnnSegmentFile.java +++ b/paimon-core/src/main/java/org/apache/paimon/index/pkvector/PkVectorAnnSegmentFile.java @@ -52,8 +52,6 @@ /** Builds immutable ANN payloads whose index ids are source data-file row positions. */ public class PkVectorAnnSegmentFile extends IndexFile { - public static final String PK_VECTOR_ANN = "pk-vector-ann"; - public PkVectorAnnSegmentFile(FileIO fileIO, IndexPathFactory pathFactory) { super(fileIO, pathFactory); } @@ -148,16 +146,19 @@ public IndexFileMeta build( ResultEntry result = results.get(0); Path payloadPath = fileWriter.path(result.fileName()); byte[] payloadMetadata = result.meta() == null ? new byte[0] : result.meta(); - PkVectorAnnSegmentMeta metadata = - new PkVectorAnnSegmentMeta(indexType, sourceFiles, payloadMetadata); IndexFileMeta segment = new IndexFileMeta( - PK_VECTOR_ANN, + indexType, result.fileName(), fileIO.getFileSize(payloadPath), liveRowCount, new GlobalIndexMeta( - 0, totalRowCount, vectorField.id(), null, metadata.serialize()), + 0, + totalRowCount, + vectorField.id(), + null, + payloadMetadata, + new PkVectorSourceMeta(sourceFiles).serialize()), pathFactory.isExternalPath() ? payloadPath.toString() : null); success = true; return segment; diff --git a/paimon-core/src/main/java/org/apache/paimon/index/pkvector/PkVectorAnnSegmentSearcher.java b/paimon-core/src/main/java/org/apache/paimon/index/pkvector/PkVectorAnnSegmentSearcher.java index a8f69cf7c26e..e3b2fbba7f1a 100644 --- a/paimon-core/src/main/java/org/apache/paimon/index/pkvector/PkVectorAnnSegmentSearcher.java +++ b/paimon-core/src/main/java/org/apache/paimon/index/pkvector/PkVectorAnnSegmentSearcher.java @@ -25,6 +25,7 @@ import org.apache.paimon.globalindex.GlobalIndexer; import org.apache.paimon.globalindex.ScoredGlobalIndexResult; import org.apache.paimon.globalindex.VectorGlobalIndexer; +import org.apache.paimon.index.GlobalIndexMeta; import org.apache.paimon.index.IndexFileMeta; import org.apache.paimon.options.Options; import org.apache.paimon.predicate.VectorSearch; @@ -79,7 +80,7 @@ public PkVectorAnnSegmentSearcher( public List search( IndexFileMeta segment, - PkVectorAnnSegmentMeta metadata, + PkVectorSourceMeta sourceMeta, float[] query, int limit, @Nullable DeletionVector deletionVector, @@ -87,31 +88,32 @@ public List search( Map deletionVectors = new HashMap<>(); if (deletionVector != null) { checkArgument( - metadata.sourceFiles().size() == 1, + sourceMeta.sourceFiles().size() == 1, "A single deletion vector can only search a single-source ANN segment."); - deletionVectors.put(metadata.sourceFiles().get(0).fileName(), deletionVector); + deletionVectors.put(sourceMeta.sourceFiles().get(0).fileName(), deletionVector); } - return search(segment, metadata, query, limit, deletionVectors, searchOptions); + return search(segment, sourceMeta, query, limit, deletionVectors, searchOptions); } public List search( IndexFileMeta segment, - PkVectorAnnSegmentMeta metadata, + PkVectorSourceMeta sourceMeta, float[] query, int limit, Map deletionVectors, Map searchOptions) { checkArgument(limit > 0, "Vector search limit must be positive: %s.", limit); + GlobalIndexMeta globalIndexMeta = segment.globalIndexMeta(); checkArgument( - PkVectorAnnSegmentFile.PK_VECTOR_ANN.equals(segment.indexType()), - "Vector segment %s is not an ANN payload.", + globalIndexMeta != null && globalIndexMeta.sourceMeta() != null, + "Vector segment %s has no source metadata.", segment.fileName()); GlobalIndexer indexer = - GlobalIndexer.create(metadata.indexType(), vectorField, indexOptions); + GlobalIndexer.create(segment.indexType(), vectorField, indexOptions); checkArgument( indexer instanceof VectorGlobalIndexer, "Index algorithm %s does not implement VectorGlobalIndexer.", - metadata.indexType()); + segment.indexType()); String readerMetric = normalizeMetric(((VectorGlobalIndexer) indexer).metric()); checkArgument( metric.equals(readerMetric), @@ -123,7 +125,7 @@ public List search( new GlobalIndexIOMeta( annSegmentFile.path(segment), segment.fileSize(), - metadata.payloadMetadata()); + globalIndexMeta.indexMeta()); GlobalIndexReader reader = indexer.createReader( meta -> fileIO.newInputStream(meta.filePath()), @@ -132,7 +134,7 @@ public List search( try { VectorSearch search = new VectorSearch(query, limit, vectorField.name(), searchOptions); RoaringNavigableMap64 liveRows = - liveRowPositions(metadata.sourceFiles(), deletionVectors); + liveRowPositions(sourceMeta.sourceFiles(), deletionVectors); if (liveRows != null) { search.withIncludeRowIds(liveRows); } @@ -141,7 +143,7 @@ public List search( return Collections.emptyList(); } - long sourceRowCount = totalRowCount(metadata.sourceFiles()); + long sourceRowCount = totalRowCount(sourceMeta.sourceFiles()); List candidates = new ArrayList<>(); ScoredGlobalIndexResult scored = result.get(); for (long ordinal : scored.results()) { @@ -151,7 +153,7 @@ public List search( segment.fileName(), ordinal, sourceRowCount); - FilePosition filePosition = filePosition(metadata.sourceFiles(), ordinal); + FilePosition filePosition = filePosition(sourceMeta.sourceFiles(), ordinal); DeletionVector deletionVector = deletionVectors.get(filePosition.dataFileName); checkArgument( deletionVector == null diff --git a/paimon-core/src/main/java/org/apache/paimon/index/pkvector/PkVectorAnnSegmentMeta.java b/paimon-core/src/main/java/org/apache/paimon/index/pkvector/PkVectorSourceMeta.java similarity index 55% rename from paimon-core/src/main/java/org/apache/paimon/index/pkvector/PkVectorAnnSegmentMeta.java rename to paimon-core/src/main/java/org/apache/paimon/index/pkvector/PkVectorSourceMeta.java index 368ca2575a5a..d0a6a8171ca4 100644 --- a/paimon-core/src/main/java/org/apache/paimon/index/pkvector/PkVectorAnnSegmentMeta.java +++ b/paimon-core/src/main/java/org/apache/paimon/index/pkvector/PkVectorSourceMeta.java @@ -18,90 +18,74 @@ package org.apache.paimon.index.pkvector; +import org.apache.paimon.index.GlobalIndexMeta; +import org.apache.paimon.index.IndexFileMeta; import org.apache.paimon.io.DataInputDeserializer; import org.apache.paimon.io.DataOutputSerializer; import java.io.IOException; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collections; import java.util.List; -import java.util.Objects; import static org.apache.paimon.utils.Preconditions.checkArgument; -/** Versioned metadata for an immutable ANN primary-key vector segment. */ -public final class PkVectorAnnSegmentMeta { +/** Ordered source data files for a primary-key vector index payload. */ +public final class PkVectorSourceMeta { private static final int VERSION = 1; - private final String indexType; private final List sourceFiles; - private final byte[] payloadMetadata; - public PkVectorAnnSegmentMeta( - String indexType, List sourceFiles, byte[] payloadMetadata) { - this.indexType = Objects.requireNonNull(indexType); + public PkVectorSourceMeta(List sourceFiles) { this.sourceFiles = Collections.unmodifiableList(new ArrayList<>(sourceFiles)); - this.payloadMetadata = Arrays.copyOf(payloadMetadata, payloadMetadata.length); - checkArgument(!this.sourceFiles.isEmpty(), "An ANN segment must reference source files."); - } - - public String indexType() { - return indexType; + checkArgument(!this.sourceFiles.isEmpty(), "A vector index must reference source files."); } public List sourceFiles() { return sourceFiles; } - public byte[] payloadMetadata() { - return Arrays.copyOf(payloadMetadata, payloadMetadata.length); + public static PkVectorSourceMeta fromIndexFile(IndexFileMeta indexFile) { + GlobalIndexMeta globalIndexMeta = indexFile.globalIndexMeta(); + checkArgument( + globalIndexMeta != null && globalIndexMeta.sourceMeta() != null, + "Vector index file %s has no source metadata.", + indexFile.fileName()); + return deserialize(globalIndexMeta.sourceMeta()); } public byte[] serialize() { try { DataOutputSerializer output = new DataOutputSerializer(128); output.writeInt(VERSION); - output.writeUTF(indexType); output.writeInt(sourceFiles.size()); for (PkVectorSourceFile sourceFile : sourceFiles) { output.writeUTF(sourceFile.fileName()); output.writeLong(sourceFile.rowCount()); } - output.writeInt(payloadMetadata.length); - output.write(payloadMetadata); return output.getCopyOfBuffer(); } catch (IOException e) { - throw new RuntimeException("Failed to serialize ANN vector segment metadata.", e); + throw new RuntimeException("Failed to serialize vector source metadata.", e); } } - public static PkVectorAnnSegmentMeta deserialize(byte[] bytes) { + public static PkVectorSourceMeta deserialize(byte[] bytes) { try { DataInputDeserializer input = new DataInputDeserializer(bytes); int version = input.readInt(); - checkArgument( - version == VERSION, "Unsupported ANN vector segment version: %s.", version); - String indexType = input.readUTF(); + checkArgument(version == VERSION, "Unsupported vector source version: %s.", version); int sourceFileCount = input.readInt(); - checkArgument(sourceFileCount > 0, "An ANN segment must reference source files."); + checkArgument(sourceFileCount > 0, "A vector index must reference source files."); List sourceFiles = new ArrayList<>(sourceFileCount); for (int i = 0; i < sourceFileCount; i++) { sourceFiles.add(new PkVectorSourceFile(input.readUTF(), input.readLong())); } - int payloadMetadataLength = input.readInt(); - checkArgument( - payloadMetadataLength >= 0, "Payload metadata length must not be negative."); - byte[] payloadMetadata = new byte[payloadMetadataLength]; - input.readFully(payloadMetadata); checkArgument( - input.available() == 0, - "Unexpected trailing bytes in ANN vector segment metadata."); - return new PkVectorAnnSegmentMeta(indexType, sourceFiles, payloadMetadata); + input.available() == 0, "Unexpected trailing bytes in vector source metadata."); + return new PkVectorSourceMeta(sourceFiles); } catch (IOException e) { - throw new IllegalArgumentException( - "Failed to deserialize ANN vector segment metadata.", e); + throw new IllegalArgumentException("Failed to deserialize vector source metadata.", e); } } } diff --git a/paimon-core/src/main/java/org/apache/paimon/manifest/IndexManifestEntrySerializer.java b/paimon-core/src/main/java/org/apache/paimon/manifest/IndexManifestEntrySerializer.java index 60113adff1c9..ef0b68d14240 100644 --- a/paimon-core/src/main/java/org/apache/paimon/manifest/IndexManifestEntrySerializer.java +++ b/paimon-core/src/main/java/org/apache/paimon/manifest/IndexManifestEntrySerializer.java @@ -60,7 +60,8 @@ public InternalRow convertTo(IndexManifestEntry record) { globalIndexMeta.extraFieldIds() == null ? null : new GenericArray(globalIndexMeta.extraFieldIds()), - globalIndexMeta.indexMeta()); + globalIndexMeta.indexMeta(), + globalIndexMeta.sourceMeta()); return GenericRow.of( record.kind().toByteValue(), serializeBinaryRow(record.partition()), @@ -82,16 +83,25 @@ public IndexManifestEntry convertFrom(int version, InternalRow row) { GlobalIndexMeta globalIndexMeta = null; if (!row.isNullAt(9)) { - InternalRow globalIndexRow = row.getRow(9, 5); + InternalRow globalIndexRow = row.getRow(9, 6); long rowRangeStart = globalIndexRow.getLong(0); long rowRangeEnd = globalIndexRow.getLong(1); int indexFieldId = globalIndexRow.getInt(2); int[] extralFields = globalIndexRow.isNullAt(3) ? null : globalIndexRow.getArray(3).toIntArray(); byte[] indexMeta = globalIndexRow.isNullAt(4) ? null : globalIndexRow.getBinary(4); + byte[] sourceMeta = + globalIndexRow.getFieldCount() <= 5 || globalIndexRow.isNullAt(5) + ? null + : globalIndexRow.getBinary(5); globalIndexMeta = new GlobalIndexMeta( - rowRangeStart, rowRangeEnd, indexFieldId, extralFields, indexMeta); + rowRangeStart, + rowRangeEnd, + indexFieldId, + extralFields, + indexMeta, + sourceMeta); } return new IndexManifestEntry( diff --git a/paimon-core/src/test/java/org/apache/paimon/index/IndexFileMetaSerializerTest.java b/paimon-core/src/test/java/org/apache/paimon/index/IndexFileMetaSerializerTest.java index 7e4fe92c8d32..00d97d54351e 100644 --- a/paimon-core/src/test/java/org/apache/paimon/index/IndexFileMetaSerializerTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/index/IndexFileMetaSerializerTest.java @@ -22,12 +22,35 @@ import org.apache.paimon.utils.ObjectSerializer; import org.apache.paimon.utils.ObjectSerializerTestBase; +import org.junit.jupiter.api.Test; + import java.util.LinkedHashMap; import java.util.Random; +import static org.assertj.core.api.Assertions.assertThat; + /** Test for {@link org.apache.paimon.index.IndexFileMetaSerializer}. */ public class IndexFileMetaSerializerTest extends ObjectSerializerTestBase { + @Test + void testGlobalIndexSourceMetaRoundTrip() { + IndexFileMetaSerializer serializer = new IndexFileMetaSerializer(); + IndexFileMeta indexFile = + new IndexFileMeta( + "ivf-pq", + "index-file", + 100, + 10, + new GlobalIndexMeta(0, 9, 7, null, new byte[] {3, 4}, new byte[] {1, 2}), + null); + + GlobalIndexMeta restored = + serializer.fromRow(serializer.toRow(indexFile)).globalIndexMeta(); + + assertThat(restored.sourceMeta()).containsExactly(1, 2); + assertThat(restored.indexMeta()).containsExactly(3, 4); + } + @Override protected ObjectSerializer serializer() { return new IndexFileMetaSerializer(); diff --git a/paimon-core/src/test/java/org/apache/paimon/index/pkvector/PkVectorAnnSegmentFileTest.java b/paimon-core/src/test/java/org/apache/paimon/index/pkvector/PkVectorAnnSegmentFileTest.java index 97ed4d379e86..6e25af5840a1 100644 --- a/paimon-core/src/test/java/org/apache/paimon/index/pkvector/PkVectorAnnSegmentFileTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/index/pkvector/PkVectorAnnSegmentFileTest.java @@ -67,12 +67,10 @@ void testBuildSkipsNullAndExcludedPhysicalRows() throws Exception { "l2", "test-vector-ann"); - assertThat(segment.indexType()).isEqualTo(PkVectorAnnSegmentFile.PK_VECTOR_ANN); + assertThat(segment.indexType()).isEqualTo("test-vector-ann"); assertThat(segment.rowCount()).isEqualTo(1); - PkVectorAnnSegmentMeta metadata = - PkVectorAnnSegmentMeta.deserialize(segment.globalIndexMeta().indexMeta()); - assertThat(metadata.indexType()).isEqualTo("test-vector-ann"); - assertThat(metadata.sourceFiles()) + PkVectorSourceMeta sourceMeta = PkVectorSourceMeta.fromIndexFile(segment); + assertThat(sourceMeta.sourceFiles()) .extracting(PkVectorSourceFile::fileName) .containsExactly("data-1"); } @@ -94,9 +92,7 @@ void testBuildsAndSearchesMultiSourceSegment() throws Exception { indexOptions(), "l2", "test-vector-ann"); - PkVectorAnnSegmentMeta metadata = - PkVectorAnnSegmentMeta.deserialize(segment.globalIndexMeta().indexMeta()); - assertThat(metadata.indexType()).isEqualTo("test-vector-ann"); + PkVectorSourceMeta sourceMeta = PkVectorSourceMeta.fromIndexFile(segment); BitmapDeletionVector data2Deletes = new BitmapDeletionVector(); data2Deletes.delete(0); Map deletionVectors = @@ -111,7 +107,7 @@ void testBuildsAndSearchesMultiSourceSegment() throws Exception { fileIO, annFile, vectorField(), indexOptions(), "l2", executor) .search( segment, - metadata, + sourceMeta, new float[] {0, 0}, 3, deletionVectors, diff --git a/paimon-core/src/test/java/org/apache/paimon/index/pkvector/PkVectorAnnSegmentMetaTest.java b/paimon-core/src/test/java/org/apache/paimon/index/pkvector/PkVectorSourceMetaTest.java similarity index 64% rename from paimon-core/src/test/java/org/apache/paimon/index/pkvector/PkVectorAnnSegmentMetaTest.java rename to paimon-core/src/test/java/org/apache/paimon/index/pkvector/PkVectorSourceMetaTest.java index 8284af0f2de6..9d8e96726752 100644 --- a/paimon-core/src/test/java/org/apache/paimon/index/pkvector/PkVectorAnnSegmentMetaTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/index/pkvector/PkVectorSourceMetaTest.java @@ -27,37 +27,30 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; -/** Tests for {@link PkVectorAnnSegmentMeta}. */ -class PkVectorAnnSegmentMetaTest { +/** Tests for {@link PkVectorSourceMeta}. */ +class PkVectorSourceMetaTest { @Test void testRoundTrip() { - PkVectorAnnSegmentMeta metadata = - new PkVectorAnnSegmentMeta( - "test-vector-ann", + PkVectorSourceMeta metadata = + new PkVectorSourceMeta( Arrays.asList( - new PkVectorSourceFile("data-1", 100), - new PkVectorSourceFile("data-2", 50)), - new byte[] {1, 2, 3}); + new PkVectorSourceFile("data-1", 10), + new PkVectorSourceFile("data-2", 20))); - PkVectorAnnSegmentMeta restored = PkVectorAnnSegmentMeta.deserialize(metadata.serialize()); + PkVectorSourceMeta restored = PkVectorSourceMeta.deserialize(metadata.serialize()); - assertThat(restored.indexType()).isEqualTo("test-vector-ann"); assertThat(restored.sourceFiles()).isEqualTo(metadata.sourceFiles()); - assertThat(restored.payloadMetadata()).containsExactly(1, 2, 3); } @Test - void testRejectsTruncatedPayloadMetadata() throws Exception { + void testRejectsTruncatedSourceMetadata() throws Exception { DataOutputSerializer output = new DataOutputSerializer(128); output.writeInt(1); - output.writeUTF("index"); output.writeInt(1); output.writeUTF("data-1"); - output.writeLong(10); - output.writeInt(1); - assertThatThrownBy(() -> PkVectorAnnSegmentMeta.deserialize(output.getCopyOfBuffer())) - .hasMessageContaining("Failed to deserialize ANN vector segment metadata"); + assertThatThrownBy(() -> PkVectorSourceMeta.deserialize(output.getCopyOfBuffer())) + .hasMessageContaining("Failed to deserialize vector source metadata"); } } diff --git a/paimon-core/src/test/java/org/apache/paimon/manifest/IndexManifestEntrySerializerTest.java b/paimon-core/src/test/java/org/apache/paimon/manifest/IndexManifestEntrySerializerTest.java index 0429b8dae3e0..0678ed787aaa 100644 --- a/paimon-core/src/test/java/org/apache/paimon/manifest/IndexManifestEntrySerializerTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/manifest/IndexManifestEntrySerializerTest.java @@ -18,17 +18,55 @@ package org.apache.paimon.manifest; +import org.apache.paimon.data.BinaryRow; +import org.apache.paimon.data.GenericRow; +import org.apache.paimon.data.InternalRow; +import org.apache.paimon.index.GlobalIndexMeta; +import org.apache.paimon.index.IndexFileMeta; import org.apache.paimon.utils.ObjectSerializer; import org.apache.paimon.utils.ObjectSerializerTestBase; +import org.junit.jupiter.api.Test; + import java.util.Random; import static org.apache.paimon.index.IndexFileMetaSerializerTest.randomIndexFile; import static org.apache.paimon.io.DataFileTestUtils.row; +import static org.assertj.core.api.Assertions.assertThat; /** Test for {@link IndexManifestEntrySerializer}. */ public class IndexManifestEntrySerializerTest extends ObjectSerializerTestBase { + @Test + void testReadsGlobalIndexWithoutSourceMeta() { + IndexManifestEntrySerializer serializer = new IndexManifestEntrySerializer(); + IndexManifestEntry entry = + new IndexManifestEntry( + FileKind.ADD, + BinaryRow.EMPTY_ROW, + 0, + new IndexFileMeta( + "btree", + "index-file", + 100, + 10, + new GlobalIndexMeta(0, 9, 7, null, new byte[] {1}), + null)); + GenericRow serialized = (GenericRow) serializer.convertTo(entry); + serialized.setField(9, GenericRow.of(0L, 9L, 7, null, new byte[] {1})); + + InternalRow globalIndexRow = serialized.getRow(9, 5); + assertThat(globalIndexRow.getFieldCount()).isEqualTo(5); + GlobalIndexMeta restored = + serializer + .convertFrom(serializer.getVersion(), serialized) + .indexFile() + .globalIndexMeta(); + + assertThat(restored.indexMeta()).containsExactly(1); + assertThat(restored.sourceMeta()).isNull(); + } + @Override protected ObjectSerializer serializer() { return new IndexManifestEntrySerializer(); From eb09f26c9d57d3a56f91d1efa4b542c19942d2fa Mon Sep 17 00:00:00 2001 From: JingsongLi Date: Sat, 11 Jul 2026 20:44:52 +0800 Subject: [PATCH 16/19] [core] Remove primary-key vector ANN row threshold --- .../main/java/org/apache/paimon/CoreOptions.java | 16 ++-------------- .../apache/paimon/schema/SchemaValidation.java | 3 --- .../PrimaryKeyVectorIndexOptionsTest.java | 11 ----------- .../PrimaryKeyVectorIndexValidationTest.java | 10 ---------- 4 files changed, 2 insertions(+), 38 deletions(-) diff --git a/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java b/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java index 17825da46cc2..b87233e756b1 100644 --- a/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java +++ b/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java @@ -2734,16 +2734,8 @@ public String toString() { "Comma-separated VECTOR columns indexed by primary-key vector indexes. " + "Each column owns one index and must define " + "fields..pk-vector.index.type. Index options and distance " - + "metric are also field-scoped. The ANN minimum can be overridden " - + "through fields..pk-vector.ann.min-rows. The first release " - + "supports exactly one column."); - - public static final ConfigOption PK_VECTOR_ANN_MIN_ROWS = - key("pk-vector.ann.min-rows") - .longType() - .defaultValue(10_000L) - .withDescription( - "Minimum live rows required before a bucket vector segment is built as ANN."); + + "metric are also field-scoped. The first release supports exactly " + + "one column."); @Immutable public static final ConfigOption PK_CLUSTERING_OVERRIDE = @@ -4317,10 +4309,6 @@ private T primaryKeyVectorOption(String column, ConfigOption option) { return fieldOptions.get(option); } - public long primaryKeyVectorAnnMinRows(String column) { - return primaryKeyVectorOption(column, PK_VECTOR_ANN_MIN_ROWS); - } - /** Specifies the merge engine for table with primary key. */ public enum MergeEngine implements DescribedEnum { DEDUPLICATE("deduplicate", "De-duplicate and keep the last row."), diff --git a/paimon-core/src/main/java/org/apache/paimon/schema/SchemaValidation.java b/paimon-core/src/main/java/org/apache/paimon/schema/SchemaValidation.java index 7ce227ddecca..1365083dfacd 100644 --- a/paimon-core/src/main/java/org/apache/paimon/schema/SchemaValidation.java +++ b/paimon-core/src/main/java/org/apache/paimon/schema/SchemaValidation.java @@ -953,9 +953,6 @@ private static void validatePrimaryKeyVectorIndex(TableSchema schema, CoreOption "fields.%s.pk-vector.distance.metric must be one of l2, cosine, inner_product, but is %s.", indexColumn, options.primaryKeyVectorDistanceMetric(indexColumn)); - checkArgument( - options.primaryKeyVectorAnnMinRows(indexColumn) > 0, - "pk-vector.ann.min-rows must be greater than 0."); } private static void validateSequenceField(TableSchema schema, CoreOptions options) { diff --git a/paimon-core/src/test/java/org/apache/paimon/index/pkvector/PrimaryKeyVectorIndexOptionsTest.java b/paimon-core/src/test/java/org/apache/paimon/index/pkvector/PrimaryKeyVectorIndexOptionsTest.java index 90084558fbcb..c1d72175ef16 100644 --- a/paimon-core/src/test/java/org/apache/paimon/index/pkvector/PrimaryKeyVectorIndexOptionsTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/index/pkvector/PrimaryKeyVectorIndexOptionsTest.java @@ -88,17 +88,6 @@ void testFieldScopedDistanceMetricOverridesTableDefault() { .isEqualTo("cosine"); } - @Test - void testFieldScopedAnnThresholdOverridesTableDefault() { - Map options = new HashMap<>(); - options.put(CoreOptions.PK_VECTOR_INDEX_COLUMNS.key(), "embedding"); - options.put(CoreOptions.PK_VECTOR_ANN_MIN_ROWS.key(), "10000"); - options.put("fields.embedding.pk-vector.ann.min-rows", "20000"); - - CoreOptions coreOptions = new CoreOptions(options); - assertThat(coreOptions.primaryKeyVectorAnnMinRows("embedding")).isEqualTo(20_000L); - } - @Test void testFieldScopedJsonOptionsOverrideTableDefault() { Map options = new HashMap<>(); diff --git a/paimon-core/src/test/java/org/apache/paimon/schema/PrimaryKeyVectorIndexValidationTest.java b/paimon-core/src/test/java/org/apache/paimon/schema/PrimaryKeyVectorIndexValidationTest.java index dc9f18acf2a2..0daa7bc5c473 100644 --- a/paimon-core/src/test/java/org/apache/paimon/schema/PrimaryKeyVectorIndexValidationTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/schema/PrimaryKeyVectorIndexValidationTest.java @@ -207,16 +207,6 @@ void testRejectsUnsupportedDistanceMetric() { .hasMessageContaining("l2, cosine, inner_product"); } - @Test - void testRejectsInvalidAnnMinimumRows() { - Map options = enabledOptions(); - options.put(CoreOptions.PK_VECTOR_ANN_MIN_ROWS.key(), "0"); - - assertThatThrownBy(() -> validateTableSchema(schema(options))) - .hasMessageContaining("pk-vector.ann.min-rows") - .hasMessageContaining("greater than 0"); - } - private static Map enabledOptions() { Map options = new HashMap<>(); options.put(CoreOptions.BUCKET.key(), "1"); From 2fa64072587237dd863f872ff76272c1d2de6a87 Mon Sep 17 00:00:00 2001 From: JingsongLi Date: Sat, 11 Jul 2026 20:56:53 +0800 Subject: [PATCH 17/19] [docs] Update primary-key vector index configuration --- docs/generated/core_configuration.html | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/docs/generated/core_configuration.html b/docs/generated/core_configuration.html index 0412807fd277..d22fb24cdf12 100644 --- a/docs/generated/core_configuration.html +++ b/docs/generated/core_configuration.html @@ -1205,17 +1205,11 @@ Boolean Enables clustering by non-primary key fields. When set to true, the physical sort order of data files is determined by the configured 'clustering.columns' instead of the primary key, optimizing query performance for non-PK columns. - -
pk-vector.ann.min-rows
- 10000 - Long - Minimum live rows required before a bucket vector segment is built as ANN. -
pk-vector.index.columns
(none) String - Comma-separated VECTOR columns indexed by primary-key vector indexes. Each column owns one index and must define fields.<column>.pk-vector.index.type. Index options and distance metric are also field-scoped. The ANN minimum can be overridden through fields.<column>.pk-vector.ann.min-rows. The first release supports exactly one column. + Comma-separated VECTOR columns indexed by primary-key vector indexes. Each column owns one index and must define fields.<column>.pk-vector.index.type. Index options and distance metric are also field-scoped. The first release supports exactly one column.
postpone.batch-write-fixed-bucket
From adac9bbbd29a01301ecaca889be5478353f01d99 Mon Sep 17 00:00:00 2001 From: JingsongLi Date: Sat, 11 Jul 2026 21:35:03 +0800 Subject: [PATCH 18/19] [core] Resolve primary-key vector options in CoreOptions --- .../java/org/apache/paimon/CoreOptions.java | 93 ++++++++++++--- .../PrimaryKeyVectorIndexOptions.java | 112 ------------------ .../paimon/schema/SchemaValidation.java | 3 +- .../PrimaryKeyVectorIndexOptionsTest.java | 23 +++- 4 files changed, 96 insertions(+), 135 deletions(-) delete mode 100644 paimon-core/src/main/java/org/apache/paimon/index/pkvector/PrimaryKeyVectorIndexOptions.java diff --git a/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java b/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java index b87233e756b1..0dd64d8d2543 100644 --- a/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java +++ b/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java @@ -33,6 +33,7 @@ import org.apache.paimon.options.description.DescribedEnum; import org.apache.paimon.options.description.Description; import org.apache.paimon.options.description.InlineElement; +import org.apache.paimon.utils.JsonSerdeUtil; import org.apache.paimon.utils.MathUtils; import org.apache.paimon.utils.Pair; import org.apache.paimon.utils.StringUtils; @@ -48,11 +49,13 @@ import java.util.Collections; import java.util.HashMap; import java.util.HashSet; +import java.util.LinkedHashMap; import java.util.List; import java.util.Locale; import java.util.Map; import java.util.Optional; import java.util.Set; +import java.util.TreeMap; import java.util.UUID; import java.util.stream.Collectors; @@ -4277,16 +4280,89 @@ public List primaryKeyVectorIndexColumns() { return Arrays.stream(columns.split(",", -1)).map(String::trim).collect(Collectors.toList()); } + public String primaryKeyVectorIndexColumn() { + List columns = primaryKeyVectorIndexColumns(); + checkArgument( + columns.size() == 1, + "pk-vector.index.columns must contain exactly one column in the first release, but is %s.", + columns); + return columns.get(0); + } + @Nullable public String primaryKeyVectorIndexType(String column) { return options.get("fields." + column + ".pk-vector.index.type"); } @Nullable - public String primaryKeyVectorIndexOptions(String column) { + private String primaryKeyVectorIndexOptionsJson(String column) { return options.get("fields." + column + ".pk-vector.index.options"); } + public Options primaryKeyVectorIndexOptions(String column) { + Options resolved = new Options(toConfiguration().toMap()); + for (Map.Entry option : + primaryKeyVectorAlgorithmOptions(column).entrySet()) { + resolved.setString(option.getKey(), option.getValue()); + } + return resolved; + } + + private Map primaryKeyVectorAlgorithmOptions(String column) { + String indexTypeKey = "fields." + column + ".pk-vector.index.type"; + String indexOptionsKey = "fields." + column + ".pk-vector.index.options"; + String algorithm = primaryKeyVectorIndexType(column); + checkArgument( + algorithm != null && !algorithm.trim().isEmpty(), + "%s must be configured before resolving index options.", + indexTypeKey); + TreeMap algorithmOptions = new TreeMap<>(); + String algorithmPrefix = algorithm + "."; + String fieldPrefix = "fields." + column + "."; + for (Map.Entry entry : toConfiguration().toMap().entrySet()) { + if (entry.getKey().startsWith(algorithmPrefix) + || (entry.getKey().startsWith(fieldPrefix) + && !entry.getKey().startsWith(fieldPrefix + "pk-vector."))) { + algorithmOptions.put(entry.getKey(), entry.getValue()); + } + } + String serialized = primaryKeyVectorIndexOptionsJson(column); + if (serialized != null && !serialized.trim().isEmpty()) { + LinkedHashMap parsed; + try { + parsed = JsonSerdeUtil.parseJsonMap(serialized, String.class); + } catch (RuntimeException e) { + throw new IllegalArgumentException( + indexOptionsKey + " must be a JSON object of option key-value pairs.", e); + } + for (Map.Entry entry : parsed.entrySet()) { + String key = entry.getKey(); + String value = entry.getValue(); + checkArgument( + key != null && !key.trim().isEmpty(), + "%s contains an empty option key.", + indexOptionsKey); + checkArgument( + value != null, + "%s value for key %s must not be null.", + indexOptionsKey, + key); + String qualifiedKey = + key.startsWith(algorithmPrefix) || key.startsWith("fields.") + ? key + : algorithmPrefix + key; + String previous = algorithmOptions.put(qualifiedKey, value); + checkArgument( + previous == null || previous.equals(value), + "%s defines conflicting values for %s.", + indexOptionsKey, + qualifiedKey); + } + } + algorithmOptions.put(algorithmPrefix + "metric", primaryKeyVectorDistanceMetric(column)); + return algorithmOptions; + } + public String primaryKeyVectorDistanceMetric(String column) { String metric = options.get("fields." + column + ".pk-vector.distance.metric"); return (metric == null ? "inner_product" : metric) @@ -4294,21 +4370,6 @@ public String primaryKeyVectorDistanceMetric(String column) { .replace('-', '_'); } - @Nullable - private String primaryKeyVectorFieldOption(String column, ConfigOption option) { - return options.get("fields." + column + "." + option.key()); - } - - private T primaryKeyVectorOption(String column, ConfigOption option) { - String fieldValue = primaryKeyVectorFieldOption(column, option); - if (fieldValue == null) { - return options.get(option); - } - Options fieldOptions = new Options(); - fieldOptions.setString(option.key(), fieldValue); - return fieldOptions.get(option); - } - /** Specifies the merge engine for table with primary key. */ public enum MergeEngine implements DescribedEnum { DEDUPLICATE("deduplicate", "De-duplicate and keep the last row."), diff --git a/paimon-core/src/main/java/org/apache/paimon/index/pkvector/PrimaryKeyVectorIndexOptions.java b/paimon-core/src/main/java/org/apache/paimon/index/pkvector/PrimaryKeyVectorIndexOptions.java deleted file mode 100644 index 46ea22ad6ed1..000000000000 --- a/paimon-core/src/main/java/org/apache/paimon/index/pkvector/PrimaryKeyVectorIndexOptions.java +++ /dev/null @@ -1,112 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.paimon.index.pkvector; - -import org.apache.paimon.CoreOptions; -import org.apache.paimon.options.Options; -import org.apache.paimon.utils.JsonSerdeUtil; - -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.TreeMap; - -import static org.apache.paimon.utils.Preconditions.checkArgument; - -/** Resolves algorithm options for the primary-key vector index. */ -public final class PrimaryKeyVectorIndexOptions { - - private PrimaryKeyVectorIndexOptions() {} - - public static Options resolve(CoreOptions coreOptions) { - return resolve(coreOptions, singleColumn(coreOptions)); - } - - public static Options resolve(CoreOptions coreOptions, String field) { - Options resolved = new Options(coreOptions.toConfiguration().toMap()); - for (Map.Entry option : algorithmOptions(coreOptions, field).entrySet()) { - resolved.setString(option.getKey(), option.getValue()); - } - return resolved; - } - - public static String singleColumn(CoreOptions coreOptions) { - List columns = coreOptions.primaryKeyVectorIndexColumns(); - checkArgument( - columns.size() == 1, - "pk-vector.index.columns must contain exactly one column in the first release, but is %s.", - columns); - return columns.get(0); - } - - private static Map algorithmOptions(CoreOptions coreOptions, String field) { - String indexTypeKey = "fields." + field + ".pk-vector.index.type"; - String indexOptionsKey = "fields." + field + ".pk-vector.index.options"; - String algorithm = coreOptions.primaryKeyVectorIndexType(field); - checkArgument( - algorithm != null && !algorithm.trim().isEmpty(), - "%s must be configured before resolving index options.", - indexTypeKey); - TreeMap options = new TreeMap<>(); - String algorithmPrefix = algorithm + "."; - String fieldPrefix = "fields." + field + "."; - for (Map.Entry entry : coreOptions.toConfiguration().toMap().entrySet()) { - if (entry.getKey().startsWith(algorithmPrefix) - || (entry.getKey().startsWith(fieldPrefix) - && !entry.getKey().startsWith(fieldPrefix + "pk-vector."))) { - options.put(entry.getKey(), entry.getValue()); - } - } - String serialized = coreOptions.primaryKeyVectorIndexOptions(field); - if (serialized != null && !serialized.trim().isEmpty()) { - LinkedHashMap parsed; - try { - parsed = JsonSerdeUtil.parseJsonMap(serialized, String.class); - } catch (RuntimeException e) { - throw new IllegalArgumentException( - indexOptionsKey + " must be a JSON object of option key-value pairs.", e); - } - for (Map.Entry entry : parsed.entrySet()) { - String key = entry.getKey(); - String value = entry.getValue(); - checkArgument( - key != null && !key.trim().isEmpty(), - "%s contains an empty option key.", - indexOptionsKey); - checkArgument( - value != null, - "%s value for key %s must not be null.", - indexOptionsKey, - key); - String qualifiedKey = - key.startsWith(algorithmPrefix) || key.startsWith("fields.") - ? key - : algorithmPrefix + key; - String previous = options.put(qualifiedKey, value); - checkArgument( - previous == null || previous.equals(value), - "%s defines conflicting values for %s.", - indexOptionsKey, - qualifiedKey); - } - } - options.put(algorithmPrefix + "metric", coreOptions.primaryKeyVectorDistanceMetric(field)); - return options; - } -} diff --git a/paimon-core/src/main/java/org/apache/paimon/schema/SchemaValidation.java b/paimon-core/src/main/java/org/apache/paimon/schema/SchemaValidation.java index 1365083dfacd..2122c04ed515 100644 --- a/paimon-core/src/main/java/org/apache/paimon/schema/SchemaValidation.java +++ b/paimon-core/src/main/java/org/apache/paimon/schema/SchemaValidation.java @@ -29,7 +29,6 @@ import org.apache.paimon.fileindex.FileIndexerFactory; import org.apache.paimon.fileindex.FileIndexerFactoryUtils; import org.apache.paimon.format.FileFormat; -import org.apache.paimon.index.pkvector.PrimaryKeyVectorIndexOptions; import org.apache.paimon.mergetree.compact.aggregate.FieldAggregator; import org.apache.paimon.mergetree.compact.aggregate.factory.FieldAggregatorFactory; import org.apache.paimon.options.ConfigOption; @@ -931,7 +930,7 @@ private static void validatePrimaryKeyVectorIndex(TableSchema schema, CoreOption checkArgument( !options.pkClusteringOverride(), "Primary-key vector index does not support pk-clustering-override."); - PrimaryKeyVectorIndexOptions.resolve(options, indexColumn); + options.primaryKeyVectorIndexOptions(indexColumn); DataField vectorField = schema.fields().stream() diff --git a/paimon-core/src/test/java/org/apache/paimon/index/pkvector/PrimaryKeyVectorIndexOptionsTest.java b/paimon-core/src/test/java/org/apache/paimon/index/pkvector/PrimaryKeyVectorIndexOptionsTest.java index c1d72175ef16..449d640aa3f6 100644 --- a/paimon-core/src/test/java/org/apache/paimon/index/pkvector/PrimaryKeyVectorIndexOptionsTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/index/pkvector/PrimaryKeyVectorIndexOptionsTest.java @@ -29,7 +29,7 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; -/** Tests for {@link PrimaryKeyVectorIndexOptions}. */ +/** Tests for primary-key vector index options in {@link CoreOptions}. */ class PrimaryKeyVectorIndexOptionsTest { @Test @@ -40,6 +40,14 @@ void testPluralFieldRegistryEnablesIndex() { assertThat(new CoreOptions(options).primaryKeyVectorIndexEnabled()).isTrue(); } + @Test + void testResolvesSingleIndexColumn() { + Map options = new HashMap<>(); + options.put(CoreOptions.PK_VECTOR_INDEX_COLUMNS.key(), "embedding"); + + assertThat(new CoreOptions(options).primaryKeyVectorIndexColumn()).isEqualTo("embedding"); + } + @Test void testFieldRegistryIsTheOnlyEnableSwitch() { Map options = new HashMap<>(); @@ -62,9 +70,14 @@ void testIndexTypeMustBeFieldScoped() { void testIndexOptionsMustBeFieldScoped() { Map options = new HashMap<>(); options.put(CoreOptions.PK_VECTOR_INDEX_COLUMNS.key(), "embedding"); + options.put("fields.embedding.pk-vector.index.type", "ivf-pq"); options.put("pk-vector.index.options", "{\"nlist\":64}"); - assertThat(new CoreOptions(options).primaryKeyVectorIndexOptions("embedding")).isNull(); + assertThat( + new CoreOptions(options) + .primaryKeyVectorIndexOptions("embedding") + .get("ivf-pq.nlist")) + .isNull(); } @Test @@ -96,7 +109,7 @@ void testFieldScopedJsonOptionsOverrideTableDefault() { options.put("pk-vector.index.options", "{\"nlist\":64}"); options.put("fields.embedding.pk-vector.index.options", "{\"nlist\":128}"); - Options resolved = PrimaryKeyVectorIndexOptions.resolve(new CoreOptions(options)); + Options resolved = new CoreOptions(options).primaryKeyVectorIndexOptions("embedding"); assertThat(resolved.get("ivf-pq.nlist")).isEqualTo("128"); } @@ -107,7 +120,7 @@ void testResolvesShortAndQualifiedAlgorithmOptions() { coreOptions( "{\"nlist\":64,\"ivf-pq.pq.m\":\"8\"," + "\"fields.embedding.hnsw.m\":16}"); - Options resolved = PrimaryKeyVectorIndexOptions.resolve(coreOptions); + Options resolved = coreOptions.primaryKeyVectorIndexOptions("embedding"); assertThat(resolved.get("ivf-pq.nlist")).isEqualTo("64"); assertThat(resolved.get("ivf-pq.pq.m")).isEqualTo("8"); @@ -117,7 +130,7 @@ void testResolvesShortAndQualifiedAlgorithmOptions() { @Test void testRejectsNonObjectOptions() { - assertThatThrownBy(() -> PrimaryKeyVectorIndexOptions.resolve(coreOptions("[1,2]"))) + assertThatThrownBy(() -> coreOptions("[1,2]").primaryKeyVectorIndexOptions("embedding")) .isInstanceOf(IllegalArgumentException.class) .hasMessageContaining("pk-vector.index.options") .hasMessageContaining("JSON object"); From 0415206cf577607b724ac0d02f92685b53e1aeca Mon Sep 17 00:00:00 2001 From: JingsongLi Date: Sat, 11 Jul 2026 21:55:52 +0800 Subject: [PATCH 19/19] [core] Fix primary-key vector ANN row range --- .../pkvector/PkVectorAnnSegmentFile.java | 3 ++- .../pkvector/PkVectorAnnSegmentFileTest.java | 23 +++++++++++++++++++ 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/paimon-core/src/main/java/org/apache/paimon/index/pkvector/PkVectorAnnSegmentFile.java b/paimon-core/src/main/java/org/apache/paimon/index/pkvector/PkVectorAnnSegmentFile.java index aada72eb3e8b..9a3306734e36 100644 --- a/paimon-core/src/main/java/org/apache/paimon/index/pkvector/PkVectorAnnSegmentFile.java +++ b/paimon-core/src/main/java/org/apache/paimon/index/pkvector/PkVectorAnnSegmentFile.java @@ -70,6 +70,7 @@ public IndexFileMeta build( totalRowCount = Math.addExact(totalRowCount, source.sourceFile.rowCount()); sourceFiles.add(source.sourceFile); } + checkArgument(totalRowCount > 0, "An ANN segment must reference at least one source row."); GlobalIndexer indexer = GlobalIndexer.create(indexType, vectorField, indexOptions); checkArgument( @@ -154,7 +155,7 @@ public IndexFileMeta build( liveRowCount, new GlobalIndexMeta( 0, - totalRowCount, + totalRowCount - 1, vectorField.id(), null, payloadMetadata, diff --git a/paimon-core/src/test/java/org/apache/paimon/index/pkvector/PkVectorAnnSegmentFileTest.java b/paimon-core/src/test/java/org/apache/paimon/index/pkvector/PkVectorAnnSegmentFileTest.java index 6e25af5840a1..1a30e207f4bd 100644 --- a/paimon-core/src/test/java/org/apache/paimon/index/pkvector/PkVectorAnnSegmentFileTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/index/pkvector/PkVectorAnnSegmentFileTest.java @@ -44,6 +44,7 @@ import java.util.concurrent.Executors; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; /** Tests ANN construction from generic vector readers. */ class PkVectorAnnSegmentFileTest { @@ -92,6 +93,8 @@ void testBuildsAndSearchesMultiSourceSegment() throws Exception { indexOptions(), "l2", "test-vector-ann"); + assertThat(segment.globalIndexMeta().rowRangeStart()).isZero(); + assertThat(segment.globalIndexMeta().rowRangeEnd()).isEqualTo(3); PkVectorSourceMeta sourceMeta = PkVectorSourceMeta.fromIndexFile(segment); BitmapDeletionVector data2Deletes = new BitmapDeletionVector(); data2Deletes.delete(0); @@ -126,6 +129,26 @@ fileIO, annFile, vectorField(), indexOptions(), "l2", executor) org.assertj.core.groups.Tuple.tuple("data-1", 1L)); } + @Test + void testRejectsSourcesWithoutRows() { + LocalFileIO fileIO = LocalFileIO.create(); + + assertThatThrownBy( + () -> + annFile(fileIO) + .build( + Collections.singletonList( + new PkVectorAnnSegmentFile.Source( + dataFile("empty", 0), + new ArrayReader(new float[0][]))), + vectorField(), + indexOptions(), + "l2", + "test-vector-ann")) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("at least one source row"); + } + private PkVectorAnnSegmentFile annFile(LocalFileIO fileIO) { return new PkVectorAnnSegmentFile(fileIO, pathFactory()); }