From f07a2ee96a328fe6fede1e30c71debadb11f4f54 Mon Sep 17 00:00:00 2001 From: Li Jiajia Date: Sat, 11 Jul 2026 14:38:03 -0400 Subject: [PATCH 1/7] [core] Augment scan projection with column-mask input columns A cross-column mask (e.g. display := concat_ws('-', first, last)) threw at read time when the query projected the masked target but not the mask's inputs: "Column masking refers to field 'first' which is not present in output row type". Row-filter operands are already added to the read projection and projected back out (#8447); do the same for column-mask inputs, transitively, and push the widened type before planning so column pruning keeps the files they live in. Stale rules -- columns absent from the latest schema after a rename or drop -- fail closed at plan time. The read schema is fixed once the first split reader exists, so auth-added columns no longer leak into later splits of the same TableRead; that leak also affected the existing row-filter path. Scoped to query-auth.enabled tables, except the projection resets in MergeFileSplitRead, DataEvolutionFileStoreScan and IncrementalDiffSplitRead, which run on every table. Each fixes a case where a second withReadType left the previous projection in place; the auth path is just the first caller that reconfigures a read often enough to hit it. --- .../paimon/predicate/PredicateVisitor.java | 19 +- .../paimon/catalog/TableQueryAuthResult.java | 181 +++- .../operation/DataEvolutionFileStoreScan.java | 16 +- .../paimon/operation/MergeFileSplitRead.java | 13 +- .../paimon/table/AbstractFileStoreTable.java | 1 + .../table/source/AbstractBatchTableScan.java | 4 +- .../table/source/AbstractDataTableRead.java | 164 +++- .../table/source/AbstractDataTableScan.java | 58 +- .../table/source/DataTableStreamScan.java | 4 +- .../splitread/IncrementalDiffSplitRead.java | 4 +- .../table/system/ReadOptimizedTable.java | 1 + .../catalog/TableQueryAuthResultTest.java | 44 + .../operation/MergeFileSplitReadTest.java | 154 ++- .../apache/paimon/rest/RESTCatalogTest.java | 894 ++++++++++++++++++ .../flink/lookup/LookupDataTableScan.java | 1 + .../paimon/flink/RESTCatalogITCase.java | 48 +- .../spark/SparkCatalogWithRestTest.java | 38 +- 17 files changed, 1562 insertions(+), 82 deletions(-) diff --git a/paimon-common/src/main/java/org/apache/paimon/predicate/PredicateVisitor.java b/paimon-common/src/main/java/org/apache/paimon/predicate/PredicateVisitor.java index 9f4047b8cb37..540277997ac8 100644 --- a/paimon-common/src/main/java/org/apache/paimon/predicate/PredicateVisitor.java +++ b/paimon-common/src/main/java/org/apache/paimon/predicate/PredicateVisitor.java @@ -40,6 +40,17 @@ static Set collectFieldNames(@Nullable Predicate predicate) { return predicate.visit(new FieldNameCollector()); } + /** Collects the field names referenced by a transform's inputs. */ + static Set collectFieldNames(Transform transform) { + Set fieldNames = new HashSet<>(); + for (Object input : transform.inputs()) { + if (input instanceof FieldRef) { + fieldNames.add(((FieldRef) input).name()); + } + } + return fieldNames; + } + static Set collectFieldIds(RowType rowType, @Nullable Predicate predicate) { if (predicate == null) { return Collections.emptySet(); @@ -58,13 +69,7 @@ class FieldNameCollector implements PredicateVisitor> { @Override public Set visit(LeafPredicate predicate) { - Set fieldNames = new HashSet<>(); - for (Object input : predicate.transform().inputs()) { - if (input instanceof FieldRef) { - fieldNames.add(((FieldRef) input).name()); - } - } - return fieldNames; + return collectFieldNames(predicate.transform()); } @Override diff --git a/paimon-core/src/main/java/org/apache/paimon/catalog/TableQueryAuthResult.java b/paimon-core/src/main/java/org/apache/paimon/catalog/TableQueryAuthResult.java index a2a113a5e897..0f3a55ac832e 100644 --- a/paimon-core/src/main/java/org/apache/paimon/catalog/TableQueryAuthResult.java +++ b/paimon-core/src/main/java/org/apache/paimon/catalog/TableQueryAuthResult.java @@ -28,10 +28,12 @@ import org.apache.paimon.predicate.PredicateVisitor; import org.apache.paimon.predicate.Transform; import org.apache.paimon.reader.RecordReader; +import org.apache.paimon.table.SpecialFields; import org.apache.paimon.table.source.DataFilePlan; import org.apache.paimon.table.source.QueryAuthSplit; import org.apache.paimon.table.source.Split; import org.apache.paimon.table.source.TableScan; +import org.apache.paimon.types.DataField; import org.apache.paimon.types.DataType; import org.apache.paimon.types.RowType; import org.apache.paimon.utils.InternalRowUtils; @@ -41,10 +43,15 @@ import javax.annotation.Nullable; import java.io.Serializable; +import java.util.ArrayDeque; import java.util.ArrayList; +import java.util.Deque; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Optional; +import java.util.Set; import java.util.TreeMap; import java.util.stream.Collectors; @@ -58,6 +65,10 @@ public class TableQueryAuthResult implements Serializable { private final @Nullable List filter; private final @Nullable Map columnMasking; + // lazily parsed views of the JSON rules; transient so serialization stays unchanged + private transient volatile Optional parsedFilter; + private transient volatile Map parsedMasking; + public TableQueryAuthResult( @Nullable List filter, @Nullable Map columnMasking) { this.filter = filter; @@ -74,8 +85,39 @@ public Map columnMasking() { return columnMasking; } + /** Whether this result carries any effective row-filter or masking rule. */ + public boolean hasRules() { + return extractPredicate() != null || !extractColumnMasking().isEmpty(); + } + + /** + * Widens {@code readType} with the unprojected columns the rules read, or null when the + * projection already covers them. Scans apply this before planning file pruning. + */ + @Nullable + public RowType widenReadType(RowType tableType, RowType readType) { + return appendMissingFields( + tableType, readType, requiredAuthFields(readType.getFieldNames())); + } + + /** Appends the missing {@code ruleFields} of {@code tableType} to {@code readType}. */ + @Nullable + public static RowType appendMissingFields( + RowType tableType, RowType readType, Set ruleFields) { + List widenedFields = null; + for (DataField field : tableType.getFields()) { + if (ruleFields.contains(field.name()) && !readType.containsField(field.name())) { + if (widenedFields == null) { + widenedFields = new ArrayList<>(readType.getFields()); + } + widenedFields.add(field); + } + } + return widenedFields == null ? null : readType.copy(widenedFields); + } + public TableScan.Plan convertPlan(TableScan.Plan plan) { - if (filter == null && (columnMasking == null || columnMasking.isEmpty())) { + if (!hasRules()) { return plan; } List authSplits = @@ -87,6 +129,16 @@ public TableScan.Plan convertPlan(TableScan.Plan plan) { @Nullable public Predicate extractPredicate() { + Optional parsed = parsedFilter; + if (parsed == null) { + parsed = Optional.ofNullable(parsePredicate()); + parsedFilter = parsed; + } + return parsed.orElse(null); + } + + @Nullable + private Predicate parsePredicate() { Predicate rowFilter = null; if (filter != null && !filter.isEmpty()) { List predicates = new ArrayList<>(); @@ -116,6 +168,15 @@ public static Predicate remapPredicate(Predicate predicate, RowType rowType) { } public Map extractColumnMasking() { + Map parsed = parsedMasking; + if (parsed == null) { + parsed = parseColumnMasking(); + parsedMasking = parsed; + } + return parsed; + } + + private Map parseColumnMasking() { Map result = new TreeMap<>(); if (columnMasking != null && !columnMasking.isEmpty()) { for (Map.Entry e : columnMasking.entrySet()) { @@ -131,6 +192,120 @@ public Map extractColumnMasking() { return result; } + /** + * Validates that every column the auth rules reference exists in the table's latest + * schema (not a time-travel-pinned one). A rule keyed by a since-renamed column looks just like + * an unprojected one at read time and would silently stop masking: fail closed instead. + */ + public void validateAgainstSchema(RowType tableType, @Nullable List projectedFields) { + for (Map.Entry entry : extractColumnMasking().entrySet()) { + String target = entry.getKey(); + // a mask on an unprojected system column is inert (never in the output); don't reject + if (SpecialFields.SYSTEM_FIELD_NAMES.contains(target) + && (projectedFields == null || !projectedFields.contains(target))) { + continue; + } + checkFieldExists("Column masking", target, tableType, projectedFields); + for (String input : PredicateVisitor.collectFieldNames(entry.getValue())) { + checkFieldExists("Column masking", input, tableType, projectedFields); + } + } + for (String operand : PredicateVisitor.collectFieldNames(extractPredicate())) { + checkFieldExists("Row filter", operand, tableType, projectedFields); + } + } + + /** + * Fails closed when a masked column is present in the read schema under a different name than + * the rule uses (renamed between the read snapshot and latest): name-based enforcement would + * skip the mask and leak the raw value. A column absent from the read schema is left inert. + */ + public void validateReadableWithoutRename(RowType latestType, RowType readType) { + for (Map.Entry entry : extractColumnMasking().entrySet()) { + checkNotRenamed(entry.getKey(), latestType, readType); + // a mask whose target is absent from the read schema is inert; skip its inputs + if (readType.containsField(entry.getKey())) { + for (String input : PredicateVisitor.collectFieldNames(entry.getValue())) { + checkNotRenamed(input, latestType, readType); + } + } + } + } + + private static void checkNotRenamed(String field, RowType latestType, RowType readType) { + // present by name: enforced fine. absent from latest: already thrown. else: renamed? + if (readType.containsField(field) || !latestType.containsField(field)) { + return; + } + int id = latestType.getField(field).id(); + if (readType.containsField(id)) { + throw new IllegalArgumentException( + String.format( + "Column masking references column '%s' which the snapshot being read " + + "exposes as '%s' (renamed since); refusing to read to avoid " + + "applying the rule by a stale name.", + field, readType.getField(id).name())); + } + } + + /** + * The field names the auth rules read for a query projecting {@code projectedFields}: the + * row-filter operands, plus (transitively) the inputs of every mask whose target is readable — + * projected, or itself pulled in by the filter or another mask. + */ + public Set requiredAuthFields(List projectedFields) { + Map masking = extractColumnMasking(); + Set ruleFields = new HashSet<>(); + Set readable = new HashSet<>(projectedFields); + Deque newlyReadable = new ArrayDeque<>(readable); + for (String operand : PredicateVisitor.collectFieldNames(extractPredicate())) { + ruleFields.add(operand); + if (readable.add(operand)) { + newlyReadable.add(operand); + } + } + while (!newlyReadable.isEmpty()) { + Transform mask = masking.get(newlyReadable.poll()); + if (mask == null) { + continue; + } + for (String input : PredicateVisitor.collectFieldNames(mask)) { + ruleFields.add(input); + if (readable.add(input)) { + newlyReadable.add(input); + } + } + } + return ruleFields; + } + + private static void checkFieldExists( + String rule, String field, RowType tableType, @Nullable List projectedFields) { + // system fields (e.g. _ROW_ID) are readable metadata absent from the table schema, + // but only when the query actually projects them -- they cannot be widened in + if (SpecialFields.SYSTEM_FIELD_NAMES.contains(field)) { + if (projectedFields != null && projectedFields.contains(field)) { + return; + } + throw new IllegalArgumentException( + String.format( + "%s references system column '%s' which the query does not project.", + rule, field)); + } + if (!tableType.containsField(field)) { + throw new IllegalArgumentException( + String.format( + "%s references column '%s' which does not exist in table schema %s. " + + "The rule may be stale after a column rename or drop; " + + "refusing to read.", + rule, field, tableType.getFieldNames())); + } + } + + /** + * Applies the row filter and column masking to {@code reader}. Rules are remapped by name; + * masks apply only to targets in {@code activeFields}, the columns readable from the query. + */ public RecordReader doAuth( RecordReader reader, RowType outputRowType) { return doAuth(reader, outputRowType, extractPredicate(), extractColumnMasking()); @@ -180,10 +355,6 @@ private static InternalRow transform( private static Map transformRemapping( RowType outputRowType, Map masking) { Map out = new HashMap<>(); - if (masking == null || masking.isEmpty()) { - return out; - } - for (Map.Entry e : masking.entrySet()) { String targetColumn = e.getKey(); Transform transform = e.getValue(); diff --git a/paimon-core/src/main/java/org/apache/paimon/operation/DataEvolutionFileStoreScan.java b/paimon-core/src/main/java/org/apache/paimon/operation/DataEvolutionFileStoreScan.java index 88053b03e300..5622d995d356 100644 --- a/paimon-core/src/main/java/org/apache/paimon/operation/DataEvolutionFileStoreScan.java +++ b/paimon-core/src/main/java/org/apache/paimon/operation/DataEvolutionFileStoreScan.java @@ -127,14 +127,14 @@ public DataEvolutionFileStoreScan withFilter(Predicate predicate) { @Override public FileStoreScan withReadType(RowType readType) { - if (readType != null) { - List nonSystemFields = - readType.getFields().stream() - .filter(f -> !SpecialFields.isSystemField(f.id())) - .collect(Collectors.toList()); - if (!nonSystemFields.isEmpty()) { - this.readType = readType; - } + // a type without user columns does not prune column files; reset, as this + // method may be called again + if (readType != null + && readType.getFields().stream() + .anyMatch(f -> !SpecialFields.isSystemField(f.id()))) { + this.readType = readType; + } else { + this.readType = null; } return this; } diff --git a/paimon-core/src/main/java/org/apache/paimon/operation/MergeFileSplitRead.java b/paimon-core/src/main/java/org/apache/paimon/operation/MergeFileSplitRead.java index 5e361e45239e..82eeaffdf6d1 100644 --- a/paimon-core/src/main/java/org/apache/paimon/operation/MergeFileSplitRead.java +++ b/paimon-core/src/main/java/org/apache/paimon/operation/MergeFileSplitRead.java @@ -146,10 +146,9 @@ public MergeFileSplitRead withReadType(RowType readType) { readerFactoryBuilder.withReadValueType(adjustedReadType); mergeSorter.setProjectedValueType(adjustedReadType); - // Project away fields added for merging. - if (adjustedReadType != readType) { - outerReadType = readType; - } + // Project away fields added for merging; reset any previous projection, as this + // method may be called again. + outerReadType = adjustedReadType != readType ? readType : null; return this; } @@ -516,9 +515,11 @@ public RecordReader createNoMergeReader( /** * Returns the pushed read type if {@link #withReadType(RowType)} was called, else the default - * read type. + * read type. This is the value layout the merge, comparator and serializer run on internally; + * when a sequence field was appended for merging, {@link #createMergeReader} projects its + * output back to {@code outerReadType}, so the emitted rows can be narrower than this type. */ - private RowType actualReadType() { + public RowType actualReadType() { return readerFactoryBuilder.readValueType(); } diff --git a/paimon-core/src/main/java/org/apache/paimon/table/AbstractFileStoreTable.java b/paimon-core/src/main/java/org/apache/paimon/table/AbstractFileStoreTable.java index 97058f2b8e03..fa1e00b4ea05 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/AbstractFileStoreTable.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/AbstractFileStoreTable.java @@ -290,6 +290,7 @@ public StreamDataTableScan newStreamScan() { DataTableStreamScan scan = new DataTableStreamScan( tableSchema, + schemaManager(), coreOptions(), newSnapshotReader(), snapshotManager(), diff --git a/paimon-core/src/main/java/org/apache/paimon/table/source/AbstractBatchTableScan.java b/paimon-core/src/main/java/org/apache/paimon/table/source/AbstractBatchTableScan.java index b52424b937b6..c41933221be6 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/source/AbstractBatchTableScan.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/source/AbstractBatchTableScan.java @@ -58,7 +58,6 @@ public abstract class AbstractBatchTableScan extends AbstractDataTableScan { private Integer pushDownLimit; private TopN topN; - private final SchemaManager schemaManager; @Nullable private String readProtectionTagName; protected AbstractBatchTableScan( @@ -67,10 +66,9 @@ protected AbstractBatchTableScan( CoreOptions options, SnapshotReader snapshotReader, TableQueryAuth queryAuth) { - super(schema, options, snapshotReader, queryAuth); + super(schema, schemaManager, options, snapshotReader, queryAuth); this.hasNext = true; - this.schemaManager = schemaManager; if (!schema.primaryKeys().isEmpty() && options.batchScanSkipLevel0()) { // Incremental scans read the delta or changelog files of historical snapshots, which // are always recorded at level 0. Skipping level 0 would drop all of their input. diff --git a/paimon-core/src/main/java/org/apache/paimon/table/source/AbstractDataTableRead.java b/paimon-core/src/main/java/org/apache/paimon/table/source/AbstractDataTableRead.java index 9a744a9af0be..9c20a7f8a2a7 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/source/AbstractDataTableRead.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/source/AbstractDataTableRead.java @@ -18,10 +18,10 @@ package org.apache.paimon.table.source; +import org.apache.paimon.CoreOptions; import org.apache.paimon.catalog.TableQueryAuthResult; import org.apache.paimon.data.InternalRow; import org.apache.paimon.disk.IOManager; -import org.apache.paimon.predicate.FieldRef; import org.apache.paimon.predicate.Predicate; import org.apache.paimon.predicate.PredicateProjectionConverter; import org.apache.paimon.predicate.Transform; @@ -34,7 +34,7 @@ import javax.annotation.Nullable; import java.io.IOException; -import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -42,8 +42,6 @@ import java.util.Optional; import java.util.Set; -import static org.apache.paimon.predicate.PredicateVisitor.collectFieldNames; - /** A {@link InnerTableRead} for data table. */ public abstract class AbstractDataTableRead implements InnerTableRead { @@ -52,8 +50,23 @@ public abstract class AbstractDataTableRead implements InnerTableRead { private Predicate predicate; private final TableSchema schema; - public AbstractDataTableRead(TableSchema schema) { + // The read type the subclass reads with; differs from readType only when widened for + // auth, and is fixed once a reader exists (split reads cache their format readers). + @Nullable private RowType appliedReadType; + + // blob-view columns that only resolve through the dedicated blob-view read path + private final Set resolvedBlobViewFields; + + public AbstractDataTableRead(@Nullable TableSchema schema) { this.schema = schema; + Set blobViewFields = Collections.emptySet(); + if (schema != null) { + CoreOptions options = CoreOptions.fromMap(schema.options()); + if (options.blobViewResolveEnabled()) { + blobViewFields = options.blobViewField(); + } + } + this.resolvedBlobViewFields = blobViewFields; } public abstract void applyReadType(RowType readType); @@ -90,6 +103,7 @@ public final InnerTableRead withProjection(int[] projection) { @Override public final InnerTableRead withReadType(RowType readType) { this.readType = readType; + this.appliedReadType = readType; applyReadType(readType); return this; } @@ -123,16 +137,15 @@ protected final QueryAuthContext unwrapQueryAuthSplit(Split split) { protected final RecordReader createDataReader( Split split, @Nullable TableQueryAuthResult authResult) throws IOException { - // A TableRead can be reused for multiple splits. Authentication may have expanded an - // explicitly configured physical projection for the previous split, so restore it before - // applying the current split's authorization dependencies. Without an explicit projection, - // the underlying reader must retain its own default read type. + // a TableRead is reused across splits; auth may have widened the projection for the + // previous one, so restore the requested type before deciding this split's widening if (readType != null) { applyReadType(readType); + appliedReadType = null; } RecordReader reader; if (authResult == null) { - reader = reader(split); + reader = backProject(readSplit(split)); } else { reader = authedReader(split, authResult); } @@ -143,52 +156,113 @@ protected final RecordReader createDataReader( return reader; } + private RecordReader readSplit(Split split) throws IOException { + return reader(split); + } + private RecordReader authedReader(Split split, TableQueryAuthResult authResult) throws IOException { - RecordReader reader; - RowType tableType = schema.logicalRowType(); - RowType readType = this.readType == null ? tableType : this.readType; - Predicate authPredicate = authResult.extractPredicate(); - Map columnMasking = authResult.extractColumnMasking(); - ProjectedRow backRow = null; - List readFields = readType.getFieldNames(); - Set readFieldSet = new HashSet<>(readFields); - Map selectedColumnMasking = new HashMap<>(); - for (Map.Entry mask : columnMasking.entrySet()) { - if (readFieldSet.contains(mask.getKey())) { - selectedColumnMasking.put(mask.getKey(), mask.getValue()); - } - } - Set authFields = new HashSet<>(); - if (authPredicate != null) { - authFields.addAll(collectFieldNames(authPredicate)); + List readFields = currentReadType().getFieldNames(); + Set ruleFields = authResult.requiredAuthFields(readFields); + RowType widened = widenedReadType(authResult, ruleFields); + if (widened != null && !widened.equals(appliedReadType)) { + applyReadType(widened); + appliedReadType = widened; } - for (Map.Entry mask : selectedColumnMasking.entrySet()) { - authFields.add(mask.getKey()); - for (Object input : mask.getValue().inputs()) { - if (input instanceof FieldRef) { - authFields.add(((FieldRef) input).name()); + // the split read emits appliedReadType; rules are remapped against it by name + RowType outputType = appliedReadType != null ? appliedReadType : currentReadType(); + if (widened != null && !widened.equals(outputType)) { + // rules changed after the read schema was fixed: fail clearly if they no longer fit + List outputFields = outputType.getFieldNames(); + for (String field : widened.getFieldNames()) { + if (!outputFields.contains(field)) { + throw new IllegalStateException( + String.format( + "Query auth rules changed and now require column '%s', but the " + + "read schema is already fixed to %s. Recreate the " + + "reader to apply the new rules.", + field, outputFields)); } } } - if (!authFields.isEmpty()) { - List expandedFields = new ArrayList<>(readType.getFields()); - for (DataField field : tableType.getFields()) { - if (authFields.contains(field.name()) && !readFieldSet.contains(field.name())) { - expandedFields.add(field); + // masks apply only to columns readable from the query: the ones it projects plus the + // ones the rules pulled in; a mask on anything else is inert + Map masking = authResult.extractColumnMasking(); + Map selectedMasking = Collections.emptyMap(); + if (!masking.isEmpty()) { + Set activeFields = new HashSet<>(readFields); + activeFields.addAll(ruleFields); + selectedMasking = new HashMap<>(); + for (Map.Entry mask : masking.entrySet()) { + if (activeFields.contains(mask.getKey())) { + selectedMasking.put(mask.getKey(), mask.getValue()); } } - if (expandedFields.size() > readType.getFieldCount()) { - readType = readType.copy(expandedFields); - applyReadType(readType); - backRow = ProjectedRow.from(readType.projectIndexes(readFields)); + } + RecordReader reader = + authResult.doAuth( + readSplit(split), + outputType, + authResult.extractPredicate(), + selectedMasking); + return backProject(reader); + } + + /** + * Project auth-widened rows back to the query's read type — on every split, since the widened + * read schema stays in effect even for splits without auth rules. + */ + private RecordReader backProject(RecordReader reader) { + if (appliedReadType == null || appliedReadType == readType) { + return reader; + } + ProjectedRow backRow = + ProjectedRow.from( + appliedReadType.projectIndexes(currentReadType().getFieldNames())); + return reader.transform(backRow::replaceRow); + } + + /** + * The read type widened with the unprojected columns the auth rules read (mirrors the + * row-filter augmentation of #8447), or null when the projection already covers them. The + * projected fields are kept as-is to preserve nested pruning. + */ + @Nullable + private RowType widenedReadType(TableQueryAuthResult authResult, Set ruleFields) { + RowType tableType = schema.logicalRowType(); + RowType readType = currentReadType(); + Set maskTargets = authResult.extractColumnMasking().keySet(); + for (String name : readType.getFieldNames()) { + if (!ruleFields.contains(name) && !maskTargets.contains(name)) { + continue; + } + if (!tableType.containsField(name)) { + continue; + } + // rules must not touch a nested-pruned column (partial value) + DataField tableField = tableType.getField(name); + if (!readType.getField(name).type().equals(tableField.type())) { + throw new IllegalStateException( + String.format( + "Query auth rules involve column '%s', which the query " + + "projects with a pruned type %s instead of its " + + "table type %s; cannot apply the rules to a " + + "partial column.", + name, readType.getField(name).type(), tableField.type())); } } - reader = authResult.doAuth(reader(split), readType, authPredicate, selectedColumnMasking); - if (backRow != null) { - reader = reader.transform(backRow::replaceRow); + for (String name : ruleFields) { + if (resolvedBlobViewFields.contains(name) && !readType.containsField(name)) { + // auth-added columns bypass blob-view resolution + throw new IllegalStateException( + String.format( + "Query auth rules read blob-view column '%s', which the query " + + "does not project; such columns cannot be resolved. " + + "Project the column or adjust the rule.", + name)); + } } - return reader; + return TableQueryAuthResult.appendMissingFields(tableType, readType, ruleFields); } private RecordReader executeFilter(RecordReader reader) { diff --git a/paimon-core/src/main/java/org/apache/paimon/table/source/AbstractDataTableScan.java b/paimon-core/src/main/java/org/apache/paimon/table/source/AbstractDataTableScan.java index 87257a744fae..986e9de12da1 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/source/AbstractDataTableScan.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/source/AbstractDataTableScan.java @@ -30,6 +30,7 @@ import org.apache.paimon.options.Options; import org.apache.paimon.partition.PartitionPredicate; import org.apache.paimon.predicate.Predicate; +import org.apache.paimon.schema.SchemaManager; import org.apache.paimon.schema.TableSchema; import org.apache.paimon.table.source.snapshot.CompactedStartingScanner; import org.apache.paimon.table.source.snapshot.ContinuousCompactorStartingScanner; @@ -66,6 +67,7 @@ import javax.annotation.Nullable; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Objects; @@ -83,6 +85,7 @@ abstract class AbstractDataTableScan implements DataTableScan { private static final Logger LOG = LoggerFactory.getLogger(AbstractDataTableScan.class); protected final TableSchema schema; + protected final SchemaManager schemaManager; private final CoreOptions options; protected final SnapshotReader snapshotReader; private final TableQueryAuth queryAuth; @@ -96,20 +99,26 @@ abstract class AbstractDataTableScan implements DataTableScan { protected AbstractDataTableScan( TableSchema schema, + SchemaManager schemaManager, CoreOptions options, SnapshotReader snapshotReader, TableQueryAuth queryAuth) { this.schema = schema; + this.schemaManager = schemaManager; this.options = options; this.snapshotReader = snapshotReader; this.queryAuth = queryAuth; } + // the read type last pushed to the snapshot reader; widened for auth when needed + @Nullable private RowType appliedScanReadType; + @Override public final TableScan.Plan plan() { TableQueryAuthResult queryAuthResult = authQuery(); // Always apply/clear the auth filter so removing auth leaves no stale partition pruning. applyAuthFilter(queryAuthResult == null ? null : queryAuthResult.extractPredicate()); + applyAuthReadType(queryAuthResult); Plan plan = planWithoutAuth(); if (queryAuthResult != null) { plan = queryAuthResult.convertPlan(plan); @@ -171,6 +180,7 @@ public AbstractDataTableScan withBucketFilter(Filter bucketFilter) { @Override public InnerTableScan withReadType(@Nullable RowType readType) { this.readType = readType; + this.appliedScanReadType = readType; snapshotReader.withReadType(readType); return this; } @@ -221,7 +231,21 @@ protected TableQueryAuthResult authQuery() { if (!options.queryAuthEnabled()) { return null; } - return queryAuth.auth(readType == null ? null : readType.getFieldNames()); + List select = readType == null ? null : readType.getFieldNames(); + TableQueryAuthResult result = queryAuth.auth(select); + if (result != null && result.hasRules()) { + // validated on every plan (this path already pays an auth-service call per plan), so + // schema changes under a live scan fail closed: references stale in the latest schema, + // or renamed relative to the schema being read (e.g. time travel), are rejected + RowType latestSchema = + schemaManager + .latest() + .map(TableSchema::logicalRowType) + .orElseGet(schema::logicalRowType); + result.validateAgainstSchema(latestSchema, select); + result.validateReadableWithoutRename(latestSchema, schema.logicalRowType()); + } + return result; } @Override @@ -242,6 +266,38 @@ public InnerTableScan withRowRangeIndex(RowRangeIndex rowRangeIndex) { return this; } + /** + * Push the auth-widened read type to the snapshot reader before planning, so file-level column + * pruning keeps the files of the columns the rules read. + */ + private void applyAuthReadType(@Nullable TableQueryAuthResult queryAuthResult) { + if (readType == null) { + return; + } + RowType desired = readType; + if (queryAuthResult != null && queryAuthResult.hasRules()) { + RowType widened = queryAuthResult.widenReadType(schema.logicalRowType(), readType); + if (widened != null) { + desired = widened; + } + } + // never narrow within this scan's lifetime: readers fix their schema on first use + if (appliedScanReadType != null) { + RowType widened = + TableQueryAuthResult.appendMissingFields( + appliedScanReadType, + desired, + new HashSet<>(appliedScanReadType.getFieldNames())); + if (widened != null) { + desired = widened; + } + } + if (!desired.equals(appliedScanReadType)) { + snapshotReader.withReadType(desired); + appliedScanReadType = desired; + } + } + public SnapshotReader snapshotReader() { return snapshotReader; } diff --git a/paimon-core/src/main/java/org/apache/paimon/table/source/DataTableStreamScan.java b/paimon-core/src/main/java/org/apache/paimon/table/source/DataTableStreamScan.java index 8b5031de4c1e..a7c6bf0c8890 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/source/DataTableStreamScan.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/source/DataTableStreamScan.java @@ -24,6 +24,7 @@ import org.apache.paimon.consumer.Consumer; import org.apache.paimon.manifest.PartitionEntry; import org.apache.paimon.predicate.Predicate; +import org.apache.paimon.schema.SchemaManager; import org.apache.paimon.schema.TableSchema; import org.apache.paimon.table.BucketMode; import org.apache.paimon.table.source.snapshot.AllDeltaFollowUpScanner; @@ -77,6 +78,7 @@ public class DataTableStreamScan extends AbstractDataTableScan implements Stream public DataTableStreamScan( TableSchema schema, + SchemaManager schemaManager, CoreOptions options, SnapshotReader snapshotReader, SnapshotManager snapshotManager, @@ -84,7 +86,7 @@ public DataTableStreamScan( boolean supportStreamingReadOverwrite, TableQueryAuth queryAuth, boolean hasPk) { - super(schema, options, snapshotReader, queryAuth); + super(schema, schemaManager, options, snapshotReader, queryAuth); this.options = options; this.scanMode = options.toConfiguration().get(CoreOptions.STREAM_SCAN_MODE); diff --git a/paimon-core/src/main/java/org/apache/paimon/table/source/splitread/IncrementalDiffSplitRead.java b/paimon-core/src/main/java/org/apache/paimon/table/source/splitread/IncrementalDiffSplitRead.java index d8c50d596bab..85e8bd267e1a 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/source/splitread/IncrementalDiffSplitRead.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/source/splitread/IncrementalDiffSplitRead.java @@ -113,8 +113,8 @@ public RecordReader createReader(Split s) throws IOException { mergeRead.mergeSorter(), forceKeepDelete); if (readType != null) { - ProjectedRow projectedRow = - ProjectedRow.from(readType, mergeRead.tableSchema().logicalRowType()); + // project from the merge read's actual output, which may itself be projected + ProjectedRow projectedRow = ProjectedRow.from(readType, mergeRead.actualReadType()); reader = reader.transform(kv -> kv.replaceValue(projectedRow.replaceRow(kv.value()))); } return KeyValueTableRead.unwrap(reader, mergeRead.tableSchema().options()); diff --git a/paimon-core/src/main/java/org/apache/paimon/table/system/ReadOptimizedTable.java b/paimon-core/src/main/java/org/apache/paimon/table/system/ReadOptimizedTable.java index 11bb7a353c8a..e6b7f6de7f93 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/system/ReadOptimizedTable.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/system/ReadOptimizedTable.java @@ -146,6 +146,7 @@ public StreamDataTableScan newStreamScan() { } return new DataTableStreamScan( wrapped.schema(), + schemaManager(), coreOptions(), newSnapshotReader(), snapshotManager(), diff --git a/paimon-core/src/test/java/org/apache/paimon/catalog/TableQueryAuthResultTest.java b/paimon-core/src/test/java/org/apache/paimon/catalog/TableQueryAuthResultTest.java index 3d45924871f1..ef5fb1726a25 100644 --- a/paimon-core/src/test/java/org/apache/paimon/catalog/TableQueryAuthResultTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/catalog/TableQueryAuthResultTest.java @@ -18,14 +18,21 @@ package org.apache.paimon.catalog; +import org.apache.paimon.data.BinaryString; +import org.apache.paimon.predicate.ConcatWsTransform; +import org.apache.paimon.predicate.FieldRef; import org.apache.paimon.predicate.Predicate; +import org.apache.paimon.types.DataTypes; import org.apache.paimon.types.RowType; import org.apache.paimon.utils.JsonSerdeUtil; import org.junit.jupiter.api.Test; +import java.util.Arrays; import java.util.Collections; +import java.util.Map; +import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; /** Tests that malformed query-authorization definitions cannot be silently ignored. */ @@ -84,4 +91,41 @@ void testInvalidColumnMaskFailsClosed() { .isInstanceOf(IllegalArgumentException.class) .hasMessageContaining("JSON null"); } + + private static final RowType TABLE_TYPE = + RowType.of( + new org.apache.paimon.types.DataField(0, "display", DataTypes.STRING()), + new org.apache.paimon.types.DataField(1, "extra", DataTypes.STRING())); + + private static String maskJson() { + return JsonSerdeUtil.toFlatJson( + new ConcatWsTransform( + Arrays.asList( + BinaryString.fromString("-"), + new FieldRef(1, "extra", DataTypes.STRING())))); + } + + @Test + public void testHasRules() { + assertThat(new TableQueryAuthResult(null, null).hasRules()).isFalse(); + assertThat( + new TableQueryAuthResult(Collections.emptyList(), Collections.emptyMap()) + .hasRules()) + .isFalse(); + // a blank entry is now rejected rather than ignored, see testInvalidRowFilterFailsClosed + Map masking = Collections.singletonMap("display", maskJson()); + assertThat(new TableQueryAuthResult(null, masking).hasRules()).isTrue(); + } + + @Test + public void testWidenReadType() { + Map masking = Collections.singletonMap("display", maskJson()); + TableQueryAuthResult result = new TableQueryAuthResult(null, masking); + // the mask input is unprojected: widen + RowType widened = result.widenReadType(TABLE_TYPE, TABLE_TYPE.project("display")); + assertThat(widened).isNotNull(); + assertThat(widened.getFieldNames()).containsExactly("display", "extra"); + // already covered: no widening + assertThat(result.widenReadType(TABLE_TYPE, TABLE_TYPE)).isNull(); + } } diff --git a/paimon-core/src/test/java/org/apache/paimon/operation/MergeFileSplitReadTest.java b/paimon-core/src/test/java/org/apache/paimon/operation/MergeFileSplitReadTest.java index d573b5bd905a..ce39cf6303c2 100644 --- a/paimon-core/src/test/java/org/apache/paimon/operation/MergeFileSplitReadTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/operation/MergeFileSplitReadTest.java @@ -18,6 +18,7 @@ package org.apache.paimon.operation; +import org.apache.paimon.CoreOptions; import org.apache.paimon.KeyValue; import org.apache.paimon.TestFileStore; import org.apache.paimon.TestKeyValueGenerator; @@ -44,6 +45,8 @@ import org.apache.paimon.table.PrimaryKeyTableUtils; import org.apache.paimon.table.SpecialFields; import org.apache.paimon.table.source.DataSplit; +import org.apache.paimon.table.source.IncrementalSplit; +import org.apache.paimon.table.source.splitread.IncrementalDiffSplitRead; import org.apache.paimon.types.BigIntType; import org.apache.paimon.types.DataField; import org.apache.paimon.types.DataType; @@ -323,6 +326,58 @@ record -> record.value().getString(1).toString())); } } + @Test + public void testRepeatedReadTypeResetsOuterProjection() throws Exception { + // a second withReadType that needs no adjustment must clear the outer projection + TestKeyValueGenerator gen = new TestKeyValueGenerator(); + List data = new ArrayList<>(); + for (int i = 0; i < 100; i++) { + data.add(gen.next()); + } + TestFileStore store = + createStore( + TestKeyValueGenerator.DEFAULT_PART_TYPE, + TestKeyValueGenerator.KEY_TYPE, + TestKeyValueGenerator.DEFAULT_ROW_TYPE, + TestKeyValueGenerator.TestKeyValueFieldsExtractor.EXTRACTOR, + DeduplicateMergeFunction.factory(), + Collections.singletonMap(CoreOptions.SEQUENCE_FIELD.key(), "orderId")); + store.commitData(data, gen::getPartition, kv -> 0); + + FileStoreScan scan = store.newScan(); + Long snapshotId = store.snapshotManager().latestSnapshotId(); + Map> filesGroupedByPartition = + scan.withSnapshot(snapshotId).plan().files().stream() + .collect(Collectors.groupingBy(ManifestEntry::partition)); + + MergeFileSplitRead read = store.newRead(); + // adjusted internally to include the sequence field: outer projection set + read.withReadType(TestKeyValueGenerator.DEFAULT_ROW_TYPE.project("shopId", "dt", "hr")); + // contains the sequence field, no adjustment: previous outer projection cleared + read.withReadType( + TestKeyValueGenerator.DEFAULT_ROW_TYPE.project("shopId", "dt", "hr", "orderId")); + + for (Map.Entry> entry : filesGroupedByPartition.entrySet()) { + RecordReader reader = + read.createReader( + DataSplit.builder() + .withSnapshot(snapshotId) + .withPartition(entry.getKey()) + .withBucket(0) + .withDataFiles( + entry.getValue().stream() + .map(ManifestEntry::file) + .collect(Collectors.toList())) + .withBucketPath("not used") + .build()); + RecordReaderIterator iterator = new RecordReaderIterator<>(reader); + while (iterator.hasNext()) { + assertThat(iterator.next().value().getFieldCount()).isEqualTo(4); + } + iterator.close(); + } + } + @Test public void testPostponeReader() throws Exception { RowType keyType = @@ -391,6 +446,70 @@ private static KeyValue keyValue( return new KeyValue().replace(GenericRow.of(key), sequenceNumber, kind, row); } + @Test + public void testIncrementalDiffReadOnProjectedMergeRead() throws Exception { + // the diff read projects the merge read's output; when the shared merge read + // is itself projected, the projection base must be its actual output type + TestKeyValueGenerator gen = new TestKeyValueGenerator(); + List before = new ArrayList<>(); + for (int i = 0; i < 50; i++) { + before.add(gen.next()); + } + List after = new ArrayList<>(); + for (int i = 0; i < 50; i++) { + after.add(gen.next()); + } + TestFileStore store = + createStore( + TestKeyValueGenerator.DEFAULT_PART_TYPE, + TestKeyValueGenerator.KEY_TYPE, + TestKeyValueGenerator.DEFAULT_ROW_TYPE, + TestKeyValueGenerator.TestKeyValueFieldsExtractor.EXTRACTOR, + DeduplicateMergeFunction.factory()); + store.commitData(before, gen::getPartition, kv -> 0); + store.commitData(after, gen::getPartition, kv -> 0); + + FileStoreScan scan = store.newScan(); + Long snapshotId = store.snapshotManager().latestSnapshotId(); + Map> filesByPartition = + scan.withSnapshot(snapshotId).plan().files().stream() + .collect(Collectors.groupingBy(ManifestEntry::partition)); + + MergeFileSplitRead mergeRead = store.newRead(); + // out-of-table-order projection, pushed into the shared merge read + RowType projection = TestKeyValueGenerator.DEFAULT_ROW_TYPE.project("shopId", "dt", "hr"); + mergeRead.withReadType(projection); + SplitRead diffRead = new IncrementalDiffSplitRead(mergeRead); + diffRead.withReadType(projection); + + for (Map.Entry> entry : filesByPartition.entrySet()) { + List files = + entry.getValue().stream().map(ManifestEntry::file).collect(Collectors.toList()); + IncrementalSplit split = + new IncrementalSplit( + snapshotId, + entry.getKey(), + 0, + 1, + Collections.emptyList(), + null, + files, + null, + false); + RecordReaderIterator iterator = + new RecordReaderIterator<>(diffRead.createReader(split)); + while (iterator.hasNext()) { + InternalRow row = iterator.next(); + assertThat(row.getFieldCount()).isEqualTo(3); + // shopId INT, dt STRING(len 8), hr INT: misprojection would misplace types + assertThat(row.getString(1).toString()).hasSize(8); + row.getInt(0); + row.getInt(2); + } + iterator.close(); + } + } + private List writeThenRead( List data, RowType readKeyType, @@ -446,7 +565,8 @@ private TestFileStore createStore( KeyValueFieldsExtractor extractor, MergeFunctionFactory mfFactory) throws Exception { - return createStore(1, partitionType, keyType, valueType, extractor, mfFactory); + return createStore( + 1, partitionType, keyType, valueType, extractor, mfFactory, Collections.emptyMap()); } private TestFileStore createStore( @@ -457,6 +577,36 @@ private TestFileStore createStore( KeyValueFieldsExtractor extractor, MergeFunctionFactory mfFactory) throws Exception { + return createStore( + numBuckets, + partitionType, + keyType, + valueType, + extractor, + mfFactory, + Collections.emptyMap()); + } + + private TestFileStore createStore( + RowType partitionType, + RowType keyType, + RowType valueType, + KeyValueFieldsExtractor extractor, + MergeFunctionFactory mfFactory, + Map options) + throws Exception { + return createStore(1, partitionType, keyType, valueType, extractor, mfFactory, options); + } + + private TestFileStore createStore( + int numBuckets, + RowType partitionType, + RowType keyType, + RowType valueType, + KeyValueFieldsExtractor extractor, + MergeFunctionFactory mfFactory, + Map options) + throws Exception { Path path = new Path(tempDir.toUri()); SchemaManager schemaManager = new SchemaManager(FileIOFinder.find(path), path); boolean valueCountMode = mfFactory.create() instanceof TestValueCountMergeFunction; @@ -476,7 +626,7 @@ private TestFileStore createStore( "")), partitionType.getFieldNames().stream()) .collect(Collectors.toList()), - Collections.emptyMap(), + options, null); TableSchema tableSchema = schemaManager.createTable(schema); return new TestFileStore.Builder( diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTest.java b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTest.java index 87116e3e4576..684f80105f78 100644 --- a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTest.java @@ -31,6 +31,8 @@ import org.apache.paimon.consumer.ConsumerInfo; import org.apache.paimon.consumer.ConsumerManager; import org.apache.paimon.data.BinaryString; +import org.apache.paimon.data.Blob; +import org.apache.paimon.data.BlobViewStruct; import org.apache.paimon.data.GenericRow; import org.apache.paimon.data.InternalRow; import org.apache.paimon.data.serializer.InternalRowSerializer; @@ -92,11 +94,13 @@ import org.apache.paimon.table.source.InnerTableScan; import org.apache.paimon.table.source.ReadBuilder; import org.apache.paimon.table.source.Split; +import org.apache.paimon.table.source.StreamTableScan; import org.apache.paimon.table.source.TableRead; import org.apache.paimon.table.system.SystemTableLoader; import org.apache.paimon.types.DataField; import org.apache.paimon.types.DataTypes; import org.apache.paimon.types.RowType; +import org.apache.paimon.utils.InternalRowUtils; import org.apache.paimon.utils.SnapshotManager; import org.apache.paimon.utils.SnapshotNotExistException; import org.apache.paimon.utils.StringUtils; @@ -3899,6 +3903,896 @@ void testColumnMaskingApplyOnRead() throws Exception { .isEqualTo("value"); // col5 NOT masked - original value } + private Table createMaskingAuthTable( + Identifier identifier, List fields, Map extraOptions) + throws Exception { + catalog.createDatabase(identifier.getDatabaseName(), true); + Map options = new HashMap<>(extraOptions); + options.put(QUERY_AUTH_ENABLED.key(), "true"); + catalog.createTable( + identifier, + new Schema(fields, Collections.emptyList(), Collections.emptyList(), options, ""), + true); + return catalog.getTable(identifier); + } + + private static List stringFields(String... names) { + List fields = new ArrayList<>(); + for (int i = 0; i < names.length; i++) { + fields.add(new DataField(i, names[i], DataTypes.STRING())); + } + return fields; + } + + private static void writeStringRows(Table table, String[]... rows) throws Exception { + BatchWriteBuilder writeBuilder = table.newBatchWriteBuilder(); + BatchTableWrite write = writeBuilder.newWrite(); + for (String[] values : rows) { + Object[] converted = new Object[values.length]; + for (int i = 0; i < values.length; i++) { + converted[i] = BinaryString.fromString(values[i]); + } + write.write(GenericRow.of(converted)); + } + BatchTableCommit commit = writeBuilder.newCommit(); + commit.commit(write.prepareCommit()); + write.close(); + commit.close(); + } + + private static void writeStringRow(Table table, String... values) throws Exception { + writeStringRows(table, values); + } + + /** Cross-column mask: display := concat_ws('-', first, last). */ + private void maskDisplayWithFullName(Identifier identifier) { + Map columnMasking = new HashMap<>(); + columnMasking.put( + "display", + new ConcatWsTransform( + Arrays.asList( + BinaryString.fromString("-"), + new FieldRef(0, "first", DataTypes.STRING()), + new FieldRef(1, "last", DataTypes.STRING())))); + setColumnMasking(identifier, columnMasking); + } + + /** Rows must be copied: the auth back-projection reuses one ProjectedRow per split. */ + private static List collectRows(RecordReader reader, RowType rowType) + throws Exception { + List rows = new ArrayList<>(); + reader.forEachRemaining(row -> rows.add(InternalRowUtils.copyInternalRow(row, rowType))); + return rows; + } + + @Test + void testColumnMaskingCrossColumnWithProjection() throws Exception { + Identifier identifier = + Identifier.create("test_table_db", "auth_table_masking_cross_column"); + Table table = + createMaskingAuthTable( + identifier, + stringFields("first", "last", "display", "other"), + Collections.emptyMap()); + // two rows in one commit -> one split with multiple rows + writeStringRows( + table, + new String[] {"john", "doe", "ignored", "o1"}, + new String[] {"jane", "roe", "ignored", "o2"}); + maskDisplayWithFullName(identifier); + + // project only the masked target; its input columns are not selected + ReadBuilder readBuilder = table.newReadBuilder().withProjection(new int[] {2}); + List splits = readBuilder.newScan().plan().splits(); + List rows = + collectRows( + readBuilder.newRead().createReader(splits), + table.rowType().project("display")); + + assertThat(rows).hasSize(2); + for (InternalRow row : rows) { + assertThat(row.getFieldCount()).isEqualTo(1); + } + assertThat( + rows.stream() + .map(row -> row.getString(0).toString()) + .collect(java.util.stream.Collectors.toList())) + .containsExactlyInAnyOrder("john-doe", "jane-roe"); + } + + @Test + void testColumnMaskingProjectionAcrossMultipleSplits() throws Exception { + Identifier identifier = + Identifier.create("test_table_db", "auth_table_masking_multi_split"); + Table table = + createMaskingAuthTable( + identifier, + stringFields("first", "last", "display"), + // one file per split, so the scan below yields multiple splits + Collections.singletonMap( + CoreOptions.SOURCE_SPLIT_TARGET_SIZE.key(), "1 b")); + // two commits -> two splits; the widening must not leak state across splits + writeStringRow(table, "john", "doe", "ignored"); + writeStringRow(table, "jane", "roe", "ignored"); + maskDisplayWithFullName(identifier); + + ReadBuilder readBuilder = table.newReadBuilder().withProjection(new int[] {2}); + List splits = readBuilder.newScan().plan().splits(); + assertThat(splits.size()).isGreaterThan(1); + List rows = + collectRows( + readBuilder.newRead().createReader(splits), + table.rowType().project("display")); + + List values = new ArrayList<>(); + for (InternalRow row : rows) { + // every split must be projected back to the query's arity + assertThat(row.getFieldCount()).isEqualTo(1); + values.add(row.getString(0).toString()); + } + assertThat(values).containsExactlyInAnyOrder("john-doe", "jane-roe"); + } + + @Test + void testColumnMaskingOnRowFilterColumnWithProjection() throws Exception { + Identifier identifier = + Identifier.create("test_table_db", "auth_table_masking_filter_target"); + Table table = + createMaskingAuthTable( + identifier, + stringFields("first", "last", "display", "other"), + Collections.emptyMap()); + writeStringRow(table, "john", "doe", "secret", "o1"); + + // the filter pulls unprojected "display" into the read type, activating its mask + LeafPredicate displayFilter = + LeafPredicate.of( + new FieldTransform(new FieldRef(2, "display", DataTypes.STRING())), + Equal.INSTANCE, + Collections.singletonList(BinaryString.fromString("secret"))); + setRowFilter(identifier, Collections.singletonList(displayFilter)); + maskDisplayWithFullName(identifier); + + // project only "other": the mask target and inputs are all unprojected + ReadBuilder readBuilder = table.newReadBuilder().withProjection(new int[] {3}); + List splits = readBuilder.newScan().plan().splits(); + List rows = + collectRows( + readBuilder.newRead().createReader(splits), + table.rowType().project("other")); + + assertThat(rows).hasSize(1); + assertThat(rows.get(0).getFieldCount()).isEqualTo(1); + assertThat(rows.get(0).getString(0).toString()).isEqualTo("o1"); + } + + @Test + void testColumnMaskingRevokedOnSameTableRead() throws Exception { + Identifier identifier = Identifier.create("test_table_db", "auth_table_masking_revoked"); + Table table = + createMaskingAuthTable( + identifier, + stringFields("first", "last", "display"), + Collections.emptyMap()); + writeStringRow(table, "john", "doe", "plain"); + maskDisplayWithFullName(identifier); + + ReadBuilder readBuilder = table.newReadBuilder().withProjection(new int[] {2}); + TableRead read = readBuilder.newRead(); + List masked = + collectRows( + read.createReader(readBuilder.newScan().plan().splits()), + table.rowType().project("display")); + assertThat(masked).hasSize(1); + assertThat(masked.get(0).getString(0).toString()).isEqualTo("john-doe"); + + // revoke the rules: the same TableRead must drop the widened read type + setColumnMasking(identifier, new HashMap<>()); + List plain = + collectRows( + read.createReader(readBuilder.newScan().plan().splits()), + table.rowType().project("display")); + assertThat(plain).hasSize(1); + assertThat(plain.get(0).getFieldCount()).isEqualTo(1); + assertThat(plain.get(0).getString(0).toString()).isEqualTo("plain"); + } + + @Test + void testColumnMaskingGrantedAfterReadSchemaFixed() throws Exception { + Identifier identifier = Identifier.create("test_table_db", "auth_table_masking_granted"); + Table table = + createMaskingAuthTable( + identifier, + stringFields("first", "last", "display"), + Collections.emptyMap()); + writeStringRow(table, "john", "doe", "plain"); + + // first read without rules fixes the read schema to the projection + ReadBuilder readBuilder = table.newReadBuilder().withProjection(new int[] {2}); + TableRead read = readBuilder.newRead(); + List plain = + collectRows( + read.createReader(readBuilder.newScan().plan().splits()), + table.rowType().project("display")); + assertThat(plain.get(0).getString(0).toString()).isEqualTo("plain"); + + // a mask granted afterwards needs columns outside that projection: the same reader + // widens for this split and still emits only the projected column + maskDisplayWithFullName(identifier); + List masked = + collectRows( + read.createReader(readBuilder.newScan().plan().splits()), + table.rowType().project("display")); + assertThat(masked.get(0).getFieldCount()).isEqualTo(1); + assertThat(masked.get(0).getString(0).toString()).isEqualTo("john-doe"); + } + + @Test + void testColumnMaskingPreservesNestedProjection() throws Exception { + Identifier identifier = Identifier.create("test_table_db", "auth_table_masking_nested"); + List fields = new ArrayList<>(); + fields.add(new DataField(0, "display", DataTypes.STRING())); + fields.add( + new DataField( + 1, + "s", + DataTypes.ROW( + new DataField(2, "a", DataTypes.STRING()), + new DataField(3, "b", DataTypes.STRING())))); + fields.add(new DataField(4, "extra", DataTypes.STRING())); + Table table = createMaskingAuthTable(identifier, fields, Collections.emptyMap()); + + BatchWriteBuilder writeBuilder = table.newBatchWriteBuilder(); + BatchTableWrite write = writeBuilder.newWrite(); + write.write( + GenericRow.of( + BinaryString.fromString("ignored"), + GenericRow.of(BinaryString.fromString("AV"), BinaryString.fromString("BV")), + BinaryString.fromString("EX"))); + BatchTableCommit commit = writeBuilder.newCommit(); + commit.commit(write.prepareCommit()); + write.close(); + commit.close(); + + // a nested-pruned read type, as engines push down + RowType tableRowType = table.rowType(); + DataField sField = tableRowType.getField("s"); + RowType prunedS = ((RowType) sField.type()).project("b"); + RowType prunedReadType = + new RowType( + Arrays.asList( + tableRowType.getField("display"), + new DataField(sField.id(), "s", prunedS))); + + // sanity: the nested-pruned read works without masking + ReadBuilder readBuilder = table.newReadBuilder().withReadType(prunedReadType); + List rows = + collectRows( + readBuilder.newRead().createReader(readBuilder.newScan().plan().splits()), + prunedReadType); + assertThat(rows).hasSize(1); + assertThat(rows.get(0).getRow(1, 1).getString(0).toString()).isEqualTo("BV"); + + // mask "display" from unprojected "extra": widening must keep "s" pruned + Map columnMasking = new HashMap<>(); + columnMasking.put( + "display", + new ConcatWsTransform( + Arrays.asList( + BinaryString.fromString("-"), + new FieldRef(4, "extra", DataTypes.STRING())))); + setColumnMasking(identifier, columnMasking); + + readBuilder = table.newReadBuilder().withReadType(prunedReadType); + rows = + collectRows( + readBuilder.newRead().createReader(readBuilder.newScan().plan().splits()), + prunedReadType); + assertThat(rows).hasSize(1); + assertThat(rows.get(0).getFieldCount()).isEqualTo(2); + assertThat(rows.get(0).getString(0).toString()).isEqualTo("EX"); + assertThat(rows.get(0).getRow(1, 1).getString(0).toString()).isEqualTo("BV"); + } + + @Test + void testColumnMaskingStaleRuleFailsClosed() throws Exception { + Identifier identifier = Identifier.create("test_table_db", "auth_table_masking_stale"); + Table table = + createMaskingAuthTable( + identifier, + stringFields("first", "last", "display"), + Collections.emptyMap()); + writeStringRow(table, "john", "doe", "secret"); + + // mask target absent from the schema (e.g. renamed since the rule was written) + Map staleTarget = new HashMap<>(); + staleTarget.put( + "renamed_away", + new ConcatTransform(Collections.singletonList(BinaryString.fromString("****")))); + setColumnMasking(identifier, staleTarget); + assertThatThrownBy(() -> readFully(table)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("does not exist in table schema"); + + // mask input absent from the schema + Map staleInput = new HashMap<>(); + staleInput.put( + "display", + new ConcatWsTransform( + Arrays.asList( + BinaryString.fromString("-"), + new FieldRef(0, "ghost", DataTypes.STRING())))); + setColumnMasking(identifier, staleInput); + assertThatThrownBy(() -> readFully(table)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("does not exist in table schema"); + } + + @Test + void testColumnMaskingRuleChangeOnRetainedColumn() throws Exception { + Identifier identifier = Identifier.create("test_table_db", "auth_table_masking_retained"); + Table table = + createMaskingAuthTable( + identifier, + stringFields("display", "hidden_a", "hidden_b"), + Collections.emptyMap()); + writeStringRow(table, "d1", "a1", "b1"); + + // first rules widen and fix the read schema to [display, hidden_a] + Map rules = new HashMap<>(); + rules.put( + "display", + new ConcatWsTransform( + Arrays.asList( + BinaryString.fromString("-"), + new FieldRef(1, "hidden_a", DataTypes.STRING())))); + setColumnMasking(identifier, rules); + + ReadBuilder readBuilder = table.newReadBuilder().withProjection(new int[] {0}); + TableRead read = readBuilder.newRead(); + List masked = + collectRows( + read.createReader(readBuilder.newScan().plan().splits()), + table.rowType().project("display")); + assertThat(masked.get(0).getString(0).toString()).isEqualTo("a1"); + + // the new rules mask only hidden_a, retained but unread: must not activate + rules.clear(); + rules.put( + "hidden_a", + new ConcatWsTransform( + Arrays.asList( + BinaryString.fromString("-"), + new FieldRef(2, "hidden_b", DataTypes.STRING())))); + setColumnMasking(identifier, rules); + List plain = + collectRows( + read.createReader(readBuilder.newScan().plan().splits()), + table.rowType().project("display")); + assertThat(plain).hasSize(1); + assertThat(plain.get(0).getFieldCount()).isEqualTo(1); + assertThat(plain.get(0).getString(0).toString()).isEqualTo("d1"); + } + + @Test + void testColumnMaskingRejectsNestedPrunedMaskTarget() throws Exception { + Identifier identifier = + Identifier.create("test_table_db", "auth_table_masking_pruned_target"); + List fields = new ArrayList<>(); + fields.add(new DataField(0, "display", DataTypes.STRING())); + fields.add( + new DataField( + 1, + "s", + DataTypes.ROW( + new DataField(2, "a", DataTypes.STRING()), + new DataField(3, "b", DataTypes.STRING())))); + Table table = createMaskingAuthTable(identifier, fields, Collections.emptyMap()); + + BatchWriteBuilder writeBuilder = table.newBatchWriteBuilder(); + BatchTableWrite write = writeBuilder.newWrite(); + write.write( + GenericRow.of( + BinaryString.fromString("d1"), + GenericRow.of( + BinaryString.fromString("AV"), BinaryString.fromString("BV")))); + BatchTableCommit commit = writeBuilder.newCommit(); + commit.commit(write.prepareCommit()); + write.close(); + commit.close(); + + // the mask TARGETS the struct column "s" (reading another column) + RowType tableRowType = table.rowType(); + DataField sField = tableRowType.getField("s"); + Map masking = new HashMap<>(); + masking.put( + "s", + new ConcatTransform(Collections.singletonList(BinaryString.fromString("****")))); + setColumnMasking(identifier, masking); + + // projecting "s" nested-pruned would write the mask into a partial slot: fail closed + RowType prunedS = ((RowType) sField.type()).project("b"); + RowType prunedReadType = + new RowType( + Arrays.asList( + tableRowType.getField("display"), + new DataField(sField.id(), "s", prunedS))); + ReadBuilder readBuilder = table.newReadBuilder().withReadType(prunedReadType); + assertThatThrownBy( + () -> + collectRows( + readBuilder + .newRead() + .createReader( + readBuilder.newScan().plan().splits()), + prunedReadType)) + .isInstanceOf(IllegalStateException.class) + .hasMessageContaining("pruned"); + } + + @Test + void testColumnMaskingOnColumnAddedAfterSnapshot() throws Exception { + Identifier identifier = + Identifier.create("test_table_db", "auth_table_masking_time_travel"); + Table table = + createMaskingAuthTable( + identifier, stringFields("first", "display"), Collections.emptyMap()); + writeStringRow(table, "john", "d1"); // snapshot 1 + + // add a column, then mask it: the rule is valid only in the latest schema + catalog.alterTable( + identifier, + Collections.singletonList(SchemaChange.addColumn("extra", DataTypes.STRING())), + false); + Map masking = new HashMap<>(); + masking.put( + "extra", + new ConcatTransform(Collections.singletonList(BinaryString.fromString("****")))); + setColumnMasking(identifier, masking); + + // the latest read masks the new column + Table latest = catalog.getTable(identifier); + ReadBuilder latestRead = latest.newReadBuilder(); + List latestRows = + collectRows( + latestRead.newRead().createReader(latestRead.newScan().plan().splits()), + latest.rowType()); + assertThat(latestRows).hasSize(1); + assertThat(latestRows.get(0).getString(2).toString()).isEqualTo("****"); + + // a time-travel read of the old snapshot must not fail on the newer rule + Table old = + catalog.getTable(identifier) + .copy(Collections.singletonMap(CoreOptions.SCAN_SNAPSHOT_ID.key(), "1")); + ReadBuilder oldRead = old.newReadBuilder(); + List oldRows = + collectRows( + oldRead.newRead().createReader(oldRead.newScan().plan().splits()), + old.rowType()); + assertThat(oldRows).hasSize(1); + assertThat(oldRows.get(0).getString(0).toString()).isEqualTo("john"); + } + + @Test + void testColumnMaskingRenamedColumnTimeTravelFailsClosed() throws Exception { + Identifier identifier = + Identifier.create("test_table_db", "auth_table_masking_rename_travel"); + Table table = + createMaskingAuthTable( + identifier, stringFields("first", "secret"), Collections.emptyMap()); + writeStringRow(table, "john", "s1"); // snapshot 1, column named "secret" + + // rename the column, then mask it under the new name + catalog.alterTable( + identifier, + Collections.singletonList(SchemaChange.renameColumn("secret", "masked_secret")), + false); + Map masking = new HashMap<>(); + masking.put( + "masked_secret", + new ConcatTransform(Collections.singletonList(BinaryString.fromString("****")))); + setColumnMasking(identifier, masking); + + // the latest read masks the renamed column + Table latest = catalog.getTable(identifier); + ReadBuilder latestRead = latest.newReadBuilder(); + List latestRows = + collectRows( + latestRead.newRead().createReader(latestRead.newScan().plan().splits()), + latest.rowType()); + assertThat(latestRows.get(0).getString(1).toString()).isEqualTo("****"); + + // a time-travel read of the pre-rename snapshot exposes the same physical column + // as "secret"; the rule keyed on "masked_secret" would be silently skipped by name + // and leak the raw value -- it must fail closed instead + Table old = + catalog.getTable(identifier) + .copy(Collections.singletonMap(CoreOptions.SCAN_SNAPSHOT_ID.key(), "1")); + ReadBuilder oldRead = old.newReadBuilder(); + assertThatThrownBy(() -> oldRead.newScan().plan()) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("renamed"); + } + + @Test + void testColumnMaskingSystemTargetInertWhenUnprojected() throws Exception { + Identifier identifier = + Identifier.create("test_table_db", "auth_table_masking_system_target"); + Table table = + createMaskingAuthTable( + identifier, + stringFields("display", "other"), + Collections.singletonMap(CoreOptions.ROW_TRACKING_ENABLED.key(), "true")); + writeStringRow(table, "d1", "o1"); + + // a mask on a system column the query does not project must be inert, not reject + // the whole query at plan time + Map masking = new HashMap<>(); + masking.put( + "_ROW_ID", + new ConcatTransform(Collections.singletonList(BinaryString.fromString("****")))); + setColumnMasking(identifier, masking); + + ReadBuilder readBuilder = table.newReadBuilder().withProjection(new int[] {0}); + List rows = + collectRows( + readBuilder.newRead().createReader(readBuilder.newScan().plan().splits()), + table.rowType().project("display")); + assertThat(rows).hasSize(1); + assertThat(rows.get(0).getString(0).toString()).isEqualTo("d1"); + } + + @Test + void testColumnMaskingInertTargetWithRenamedInputTimeTravel() throws Exception { + Identifier identifier = + Identifier.create("test_table_db", "auth_table_masking_inert_input"); + Table table = + createMaskingAuthTable( + identifier, stringFields("first", "old_input"), Collections.emptyMap()); + writeStringRow(table, "john", "in1"); // snapshot 1 + + // rename the input, then add a target column masked from the renamed input + catalog.alterTable( + identifier, + Collections.singletonList(SchemaChange.renameColumn("old_input", "renamed_input")), + false); + catalog.alterTable( + identifier, + Collections.singletonList(SchemaChange.addColumn("display", DataTypes.STRING())), + false); + Map masking = new HashMap<>(); + masking.put( + "display", + new FieldTransform(new FieldRef(1, "renamed_input", DataTypes.STRING()))); + setColumnMasking(identifier, masking); + + // the pre-rename snapshot predates "display": the mask cannot output there, so the + // rename of its input must not fail the read + Table old = + catalog.getTable(identifier) + .copy(Collections.singletonMap(CoreOptions.SCAN_SNAPSHOT_ID.key(), "1")); + ReadBuilder oldRead = old.newReadBuilder(); + List rows = + collectRows( + oldRead.newRead().createReader(oldRead.newScan().plan().splits()), + old.rowType()); + assertThat(rows).hasSize(1); + assertThat(rows.get(0).getString(0).toString()).isEqualTo("john"); + assertThat(rows.get(0).getString(1).toString()).isEqualTo("in1"); + } + + @Test + void testColumnMaskingRevalidatedAfterRulesDisappear() throws Exception { + Identifier identifier = Identifier.create("test_table_db", "auth_table_masking_revalidate"); + Table table = + createMaskingAuthTable( + identifier, stringFields("first", "secret"), Collections.emptyMap()); + writeStringRow(table, "john", "s1"); + + StreamTableScan scan = table.newReadBuilder().newStreamScan(); + + // plan 1: a valid mask on "secret" is validated and cached + Map masking = new HashMap<>(); + masking.put( + "secret", + new ConcatTransform(Collections.singletonList(BinaryString.fromString("****")))); + setColumnMasking(identifier, masking); + scan.plan(); + + // plan 2: rules disappear -- the cached validation must be forgotten + setColumnMasking(identifier, Collections.emptyMap()); + scan.plan(); + + // the masked column is renamed away, then the identical rule is restored; a stale-rule + // cache short-circuit would skip re-validation and silently stop masking + catalog.alterTable( + identifier, + Collections.singletonList(SchemaChange.renameColumn("secret", "hidden")), + false); + setColumnMasking(identifier, masking); + assertThatThrownBy(scan::plan) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("does not exist in table schema"); + } + + @Test + void testColumnMaskingRenamedUnderLiveScanFailsClosed() throws Exception { + Identifier identifier = + Identifier.create("test_table_db", "auth_table_masking_live_rename"); + Table table = + createMaskingAuthTable( + identifier, stringFields("first", "secret"), Collections.emptyMap()); + writeStringRow(table, "john", "s1"); + + Map masking = new HashMap<>(); + masking.put( + "secret", + new ConcatTransform(Collections.singletonList(BinaryString.fromString("****")))); + setColumnMasking(identifier, masking); + + StreamTableScan scan = table.newReadBuilder().newStreamScan(); + scan.plan(); + + // the masked column is renamed while the rules stay identical: the live scan must + // notice on its next plan and fail closed, not keep planning on the stale rule + catalog.alterTable( + identifier, + Collections.singletonList(SchemaChange.renameColumn("secret", "hidden")), + false); + assertThatThrownBy(scan::plan) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("does not exist in table schema"); + } + + @Test + void testColumnMaskingRejectsNestedPrunedRuleInput() throws Exception { + Identifier identifier = + Identifier.create("test_table_db", "auth_table_masking_pruned_input"); + List fields = new ArrayList<>(); + fields.add(new DataField(0, "display", DataTypes.STRING())); + fields.add( + new DataField( + 1, + "s", + DataTypes.ROW( + new DataField(2, "a", DataTypes.STRING()), + new DataField(3, "b", DataTypes.STRING())))); + Table table = createMaskingAuthTable(identifier, fields, Collections.emptyMap()); + + BatchWriteBuilder writeBuilder = table.newBatchWriteBuilder(); + BatchTableWrite write = writeBuilder.newWrite(); + write.write( + GenericRow.of( + BinaryString.fromString("d1"), + GenericRow.of( + BinaryString.fromString("AV"), BinaryString.fromString("BV")))); + BatchTableCommit commit = writeBuilder.newCommit(); + commit.commit(write.prepareCommit()); + write.close(); + commit.close(); + + // the mask on "display" reads the whole struct column "s" + RowType tableRowType = table.rowType(); + DataField sField = tableRowType.getField("s"); + Map masking = new HashMap<>(); + masking.put( + "display", + new CastTransform(new FieldRef(1, "s", sField.type()), DataTypes.STRING())); + setColumnMasking(identifier, masking); + + // projecting "s" nested-pruned would hand the mask a partial struct: fail closed + RowType prunedS = ((RowType) sField.type()).project("b"); + RowType prunedReadType = + new RowType( + Arrays.asList( + tableRowType.getField("display"), + new DataField(sField.id(), "s", prunedS))); + ReadBuilder readBuilder = table.newReadBuilder().withReadType(prunedReadType); + assertThatThrownBy( + () -> + collectRows( + readBuilder + .newRead() + .createReader( + readBuilder.newScan().plan().splits()), + prunedReadType)) + .isInstanceOf(IllegalStateException.class) + .hasMessageContaining("pruned"); + } + + @Test + void testColumnMaskingWithDataEvolutionColumnFiles() throws Exception { + Identifier identifier = + Identifier.create("test_table_db", "auth_table_masking_data_evolution"); + List fields = new ArrayList<>(); + fields.add(new DataField(0, "f0", DataTypes.INT())); + fields.add(new DataField(1, "f1", DataTypes.STRING())); + fields.add(new DataField(2, "f2", DataTypes.STRING())); + Map options = new HashMap<>(); + options.put(CoreOptions.ROW_TRACKING_ENABLED.key(), "true"); + options.put(CoreOptions.DATA_EVOLUTION_ENABLED.key(), "true"); + Table table = createMaskingAuthTable(identifier, fields, options); + + // one row group split across two columnar files: (f0, f1) and (f2) + RowType tableRowType = table.rowType(); + BatchWriteBuilder builder = table.newBatchWriteBuilder(); + try (BatchTableWrite write0 = + builder.newWrite().withWriteType(tableRowType.project("f0", "f1"))) { + write0.write(GenericRow.of(0, BinaryString.fromString("a0"))); + write0.write(GenericRow.of(1, BinaryString.fromString("a1"))); + builder.newCommit().commit(write0.prepareCommit()); + } + long rowId = ((FileStoreTable) table).snapshotManager().latestSnapshot().nextRowId() - 2; + builder = table.newBatchWriteBuilder(); + try (BatchTableWrite write1 = + builder.newWrite().withWriteType(tableRowType.project("f2"))) { + write1.write(GenericRow.of(BinaryString.fromString("b0"))); + write1.write(GenericRow.of(BinaryString.fromString("b1"))); + List commitables = write1.prepareCommit(); + for (CommitMessage c : commitables) { + CommitMessageImpl message = (CommitMessageImpl) c; + List newFiles = + new ArrayList<>(message.newFilesIncrement().newFiles()); + message.newFilesIncrement().newFiles().clear(); + for (DataFileMeta file : newFiles) { + message.newFilesIncrement().newFiles().add(file.assignFirstRowId(rowId)); + } + } + builder.newCommit().commit(commitables); + } + + // mask f1 from f2 and project only f1: the scan must keep f2's file + Map masking = new HashMap<>(); + masking.put( + "f1", + new ConcatWsTransform( + Arrays.asList( + BinaryString.fromString("-"), + new FieldRef(2, "f2", DataTypes.STRING())))); + setColumnMasking(identifier, masking); + + ReadBuilder readBuilder = table.newReadBuilder().withProjection(new int[] {1}); + List rows = + collectRows( + readBuilder.newRead().createReader(readBuilder.newScan().plan().splits()), + tableRowType.project("f1")); + assertThat(rows).hasSize(2); + assertThat( + rows.stream() + .map(row -> row.isNullAt(0) ? null : row.getString(0).toString()) + .collect(java.util.stream.Collectors.toList())) + .containsExactlyInAnyOrder("b0", "b1"); + } + + @Test + void testColumnMaskingRejectsUnprojectedBlobViewInput() throws Exception { + Identifier identifier = Identifier.create("test_table_db", "auth_table_masking_blob_view"); + List fields = new ArrayList<>(); + fields.add(new DataField(0, "display", DataTypes.STRING())); + fields.add(new DataField(1, "image", DataTypes.BLOB())); + Map options = new HashMap<>(); + options.put(CoreOptions.ROW_TRACKING_ENABLED.key(), "true"); + options.put(CoreOptions.DATA_EVOLUTION_ENABLED.key(), "true"); + options.put(CoreOptions.BLOB_FIELD.key(), "image"); + options.put(CoreOptions.BLOB_VIEW_FIELD.key(), "image"); + options.put(CoreOptions.BLOB_VIEW_RESOLVE_ENABLED.key(), "true"); + Table table = createMaskingAuthTable(identifier, fields, options); + + BatchWriteBuilder writeBuilder = table.newBatchWriteBuilder(); + BatchTableWrite write = writeBuilder.newWrite(); + write.write( + GenericRow.of( + BinaryString.fromString("d1"), + Blob.fromView(new BlobViewStruct(identifier, 1, 0L)))); + BatchTableCommit commit = writeBuilder.newCommit(); + commit.commit(write.prepareCommit()); + write.close(); + commit.close(); + + // the mask reads the unprojected blob-view column: resolution cannot apply + Map masking = new HashMap<>(); + masking.put( + "display", + new ConcatWsTransform( + Arrays.asList( + BinaryString.fromString("-"), + new FieldRef(1, "image", DataTypes.STRING())))); + setColumnMasking(identifier, masking); + + ReadBuilder readBuilder = table.newReadBuilder().withProjection(new int[] {0}); + assertThatThrownBy( + () -> + collectRows( + readBuilder + .newRead() + .createReader( + readBuilder.newScan().plan().splits()), + table.rowType().project("display"))) + .isInstanceOf(IllegalStateException.class) + .hasMessageContaining("blob-view"); + } + + @Test + void testColumnMaskingReadingSystemField() throws Exception { + Identifier identifier = Identifier.create("test_table_db", "auth_table_masking_row_id"); + List fields = new ArrayList<>(); + fields.add(new DataField(0, "display", DataTypes.BIGINT())); + Table table = + createMaskingAuthTable( + identifier, + fields, + Collections.singletonMap(CoreOptions.ROW_TRACKING_ENABLED.key(), "true")); + + BatchWriteBuilder writeBuilder = table.newBatchWriteBuilder(); + BatchTableWrite write = writeBuilder.newWrite(); + write.write(GenericRow.of(42L)); + BatchTableCommit commit = writeBuilder.newCommit(); + commit.commit(write.prepareCommit()); + write.close(); + commit.close(); + + // the mask reads the projected _ROW_ID metadata field: not a stale rule + Map masking = new HashMap<>(); + masking.put( + "display", + new FieldTransform(new FieldRef(0, "_ROW_ID", DataTypes.BIGINT().notNull()))); + setColumnMasking(identifier, masking); + + RowType readType = + new RowType( + Arrays.asList( + table.rowType().getField("display"), + org.apache.paimon.table.SpecialFields.ROW_ID)); + ReadBuilder readBuilder = table.newReadBuilder().withReadType(readType); + List rows = + collectRows( + readBuilder.newRead().createReader(readBuilder.newScan().plan().splits()), + readType); + assertThat(rows).hasSize(1); + // display masked to the row id + assertThat(rows.get(0).getLong(0)).isEqualTo(rows.get(0).getLong(1)); + } + + @Test + void testColumnMaskingSystemFieldValidation() throws Exception { + Identifier identifier = + Identifier.create("test_table_db", "auth_table_masking_system_validation"); + Table table = + createMaskingAuthTable( + identifier, + stringFields("display", "other"), + Collections.singletonMap(CoreOptions.ROW_TRACKING_ENABLED.key(), "true")); + writeStringRow(table, "d1", "o1"); + + // a mask keyed by a key-reader-internal name is stale, not a system field + Map masking = new HashMap<>(); + masking.put( + "_KEY_display", + new ConcatTransform(Collections.singletonList(BinaryString.fromString("****")))); + setColumnMasking(identifier, masking); + assertThatThrownBy(() -> readFully(table)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("does not exist in table schema"); + + // a rule reading an unprojected system field fails clearly at plan time + masking.clear(); + masking.put( + "display", + new FieldTransform(new FieldRef(0, "_ROW_ID", DataTypes.BIGINT().notNull()))); + setColumnMasking(identifier, masking); + ReadBuilder readBuilder = table.newReadBuilder().withProjection(new int[] {0}); + assertThatThrownBy(() -> readBuilder.newScan().plan()) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("does not project"); + } + + private static void readFully(Table table) throws Exception { + ReadBuilder readBuilder = table.newReadBuilder(); + collectRows( + readBuilder.newRead().createReader(readBuilder.newScan().plan().splits()), + table.rowType()); + } + @Test void testRowFilter() throws Exception { Identifier identifier = Identifier.create("test_table_db", "auth_table_filter"); diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/lookup/LookupDataTableScan.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/lookup/LookupDataTableScan.java index b64e7edace97..7356c670f351 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/lookup/LookupDataTableScan.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/lookup/LookupDataTableScan.java @@ -64,6 +64,7 @@ public LookupDataTableScan( LookupStreamScanMode lookupScanMode) { super( table.schema(), + table.schemaManager(), table.coreOptions(), snapshotReader, table.snapshotManager(), diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/RESTCatalogITCase.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/RESTCatalogITCase.java index 5be454a9d0c8..7e1f33136dd2 100644 --- a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/RESTCatalogITCase.java +++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/RESTCatalogITCase.java @@ -365,6 +365,51 @@ public void testColumnMasking() { Row.of("user1@example.com"), Row.of("user2@example.com")); } + @Test + public void testColumnMaskingCrossColumnWithProjection() { + String maskingTable = "cross_column_masking_table"; + batchSql( + String.format( + "CREATE TABLE %s.%s (first_name STRING, last_name STRING, display STRING, other_col STRING)" + + " WITH ('query-auth.enabled' = 'true', 'source.split.target-size' = '1 b')", + DATABASE_NAME, maskingTable)); + // two commits so the scan yields multiple splits + batchSql( + String.format( + "INSERT INTO %s.%s VALUES ('john', 'doe', 'ignored', 'o1')", + DATABASE_NAME, maskingTable)); + batchSql( + String.format( + "INSERT INTO %s.%s VALUES ('jane', 'roe', 'ignored', 'o2')", + DATABASE_NAME, maskingTable)); + + // the mask on "display" reads OTHER columns: concat_ws('-', first_name, last_name) + Map columnMasking = new HashMap<>(); + columnMasking.put( + "display", + new ConcatWsTransform( + Arrays.asList( + BinaryString.fromString("-"), + new FieldRef(0, "first_name", DataTypes.STRING()), + new FieldRef(1, "last_name", DataTypes.STRING())))); + restCatalogServer.setColumnMaskingAuth( + Identifier.create(DATABASE_NAME, maskingTable), columnMasking); + + // project only the masked target: its input columns must be read regardless + assertThat( + batchSql( + String.format( + "SELECT display FROM %s.%s", DATABASE_NAME, maskingTable))) + .containsExactlyInAnyOrder(Row.of("john-doe"), Row.of("jane-roe")); + // a projection without the masked column is unaffected + assertThat( + batchSql( + String.format( + "SELECT other_col FROM %s.%s", + DATABASE_NAME, maskingTable))) + .containsExactlyInAnyOrder(Row.of("o1"), Row.of("o2")); + } + @Test public void testRowFilter() { String filterTable = "row_filter_table"; @@ -758,7 +803,8 @@ public void testColumnMaskingAndRowFilter() { "SELECT id, name FROM %s.%s WHERE age > 30 ORDER BY id", DATABASE_NAME, combinedTable))) .rootCause() - .hasMessageContaining("Unable to read data without column non_existent_column"); + .hasMessageContaining( + "Row filter references column 'non_existent_column' which does not exist"); // Clear both column masking and row filter restCatalogServer.setColumnMaskingAuth( diff --git a/paimon-spark/paimon-spark-ut/src/test/java/org/apache/paimon/spark/SparkCatalogWithRestTest.java b/paimon-spark/paimon-spark-ut/src/test/java/org/apache/paimon/spark/SparkCatalogWithRestTest.java index fcde06202ed0..51c574857816 100644 --- a/paimon-spark/paimon-spark-ut/src/test/java/org/apache/paimon/spark/SparkCatalogWithRestTest.java +++ b/paimon-spark/paimon-spark-ut/src/test/java/org/apache/paimon/spark/SparkCatalogWithRestTest.java @@ -364,6 +364,41 @@ public void testColumnMasking() { .isEqualTo("[[user1@example.com], [user2@example.com]]"); } + @Test + public void testColumnMaskingCrossColumnWithProjection() { + spark.sql( + "CREATE TABLE t_cross_column_masking (first_name STRING, last_name STRING, display STRING, other_col STRING)" + + " TBLPROPERTIES ('query-auth.enabled'='true', 'source.split.target-size'='1 b')"); + // two commits so the scan yields multiple splits + spark.sql("INSERT INTO t_cross_column_masking VALUES ('john', 'doe', 'ignored', 'o1')"); + spark.sql("INSERT INTO t_cross_column_masking VALUES ('jane', 'roe', 'ignored', 'o2')"); + + // the mask on "display" reads OTHER columns: concat_ws('-', first_name, last_name) + Map columnMasking = new HashMap<>(); + columnMasking.put( + "display", + new ConcatWsTransform( + Arrays.asList( + BinaryString.fromString("-"), + new FieldRef(0, "first_name", DataTypes.STRING()), + new FieldRef(1, "last_name", DataTypes.STRING())))); + restCatalogServer.setColumnMaskingAuth( + Identifier.create("db2", "t_cross_column_masking"), columnMasking); + + // project only the masked target: its input columns must be read regardless + assertThat( + spark.sql("SELECT display FROM t_cross_column_masking ORDER BY other_col") + .collectAsList() + .toString()) + .isEqualTo("[[john-doe], [jane-roe]]"); + // a projection without the masked column is unaffected + assertThat( + spark.sql("SELECT other_col FROM t_cross_column_masking ORDER BY other_col") + .collectAsList() + .toString()) + .isEqualTo("[[o1], [o2]]"); + } + @Test public void testRowFilter() { spark.sql( @@ -863,7 +898,8 @@ public void testColumnMaskingAndRowFilter() { spark.sql( "SELECT id, name FROM t_combined WHERE age > 30 ORDER BY id") .collectAsList()) - .hasMessageContaining("Unable to read data without column non_existent_column"); + .hasMessageContaining( + "Row filter references column 'non_existent_column' which does not exist"); // Clear both column masking and row filter restCatalogServer.setColumnMaskingAuth( From bf9f452e8d707875697f1543112ce8b4a153a21e Mon Sep 17 00:00:00 2001 From: Li Jiajia Date: Sat, 25 Jul 2026 08:31:52 -0400 Subject: [PATCH 2/7] [core] Keep raw-value pushdown off masked columns in query auth A column mask changes the value domain of its target, so a predicate on a masked column must evaluate on the masked value. Pushed into raw statistics it matches the raw value instead and prunes away the files the query should have found -- an equality filter on a masked column returned an empty result. The filter is therefore deferred to plan(): only the conjuncts free of masked columns feed statistics and partition pruning, and the masked ones are evaluated inside the auth read, post-mask. Every column they read is widened into both the scan and the read schema -- including the unmasked operands of a disjunction, which splitAnd does not split -- and projected back out. Their presence also keeps limit/TopN split pruning off. Several planning paths reached raw values without going through that: DataEvolutionBatchScan pushed straight to the SnapshotReader and consulted the global index, PrimaryKeyBatchScan kept its own unmodified filter for the sorted indexes, and partition listing skipped the check. Where a mask cannot be enforced at all, the query is refused rather than answered from raw values, since accepting it would leave the rules silently inert: - a partition predicate on a masked partition key, which pruning consumes and Spark drops from post-scan evaluation; - the system tables reporting raw per-column statistics (files, file_key_ranges, binlog); audit_log and ro read through the masking reader and still work; - vector, full-text and hybrid search, whose indexes rank raw values; - local table queries, which serve rows straight from the lookup cache; - query-auth.enabled on a table type whose read never reaches the auth reader, rejected at create time, before and after catalog table defaults apply. A mask whose transform reads another masked column is rejected too: transforms evaluate on the raw row, so it would publish that column's raw value through its own target. Masking a column with itself stays valid. Rules bind by field id, not only by name: a dropped and re-added column keeps the name but gets a fresh id, so a time-travel read would otherwise apply a rule to unrelated historical data. Scoped to query-auth.enabled tables. --- .../paimon/predicate/PredicateVisitor.java | 6 +- .../apache/paimon/catalog/CatalogUtils.java | 11 + .../paimon/catalog/TableQueryAuthResult.java | 136 ++- .../globalindex/DataEvolutionBatchScan.java | 15 + .../operation/DataEvolutionFileStoreScan.java | 4 +- .../paimon/table/query/LocalTableQuery.java | 6 + .../table/source/AbstractBatchTableScan.java | 14 + .../table/source/AbstractDataTableRead.java | 78 +- .../table/source/AbstractDataTableScan.java | 135 ++- .../source/BatchVectorSearchBuilderImpl.java | 9 + .../table/source/DataTableStreamScan.java | 1 - .../source/FullTextSearchBuilderImpl.java | 9 + .../table/source/HybridSearchBuilderImpl.java | 10 + .../table/source/PrimaryKeyBatchScan.java | 10 +- .../paimon/table/source/ReadBuilderImpl.java | 18 +- .../table/source/VectorSearchBuilderImpl.java | 9 + .../table/system/SystemTableLoader.java | 18 +- .../catalog/TableQueryAuthResultTest.java | 51 ++ .../apache/paimon/rest/RESTCatalogTest.java | 771 +++++++++++++++++- .../paimon/flink/RESTCatalogITCase.java | 42 + .../spark/SparkCatalogWithRestTest.java | 21 + 21 files changed, 1320 insertions(+), 54 deletions(-) diff --git a/paimon-common/src/main/java/org/apache/paimon/predicate/PredicateVisitor.java b/paimon-common/src/main/java/org/apache/paimon/predicate/PredicateVisitor.java index 540277997ac8..99a6692e9e43 100644 --- a/paimon-common/src/main/java/org/apache/paimon/predicate/PredicateVisitor.java +++ b/paimon-common/src/main/java/org/apache/paimon/predicate/PredicateVisitor.java @@ -40,8 +40,8 @@ static Set collectFieldNames(@Nullable Predicate predicate) { return predicate.visit(new FieldNameCollector()); } - /** Collects the field names referenced by a transform's inputs. */ - static Set collectFieldNames(Transform transform) { + /** Collects the field names a transform's inputs reference. */ + static Set collectTransformFieldNames(Transform transform) { Set fieldNames = new HashSet<>(); for (Object input : transform.inputs()) { if (input instanceof FieldRef) { @@ -69,7 +69,7 @@ class FieldNameCollector implements PredicateVisitor> { @Override public Set visit(LeafPredicate predicate) { - return collectFieldNames(predicate.transform()); + return collectTransformFieldNames(predicate.transform()); } @Override diff --git a/paimon-core/src/main/java/org/apache/paimon/catalog/CatalogUtils.java b/paimon-core/src/main/java/org/apache/paimon/catalog/CatalogUtils.java index 0ee552d887f6..9afa62468ea9 100644 --- a/paimon-core/src/main/java/org/apache/paimon/catalog/CatalogUtils.java +++ b/paimon-core/src/main/java/org/apache/paimon/catalog/CatalogUtils.java @@ -168,6 +168,17 @@ public static void validateCreateTable(Schema schema, boolean dataTokenEnabled) if (tableType.equals(TableType.FORMAT_TABLE)) { validateFormatTableOptions(options, dataTokenEnabled); } + // only a file-store table reads through the auth reader; anywhere else the rules would + // be accepted and then silently not applied + if (options.get(CoreOptions.QUERY_AUTH_ENABLED) + && tableType != TableType.TABLE + && tableType != TableType.MATERIALIZED_TABLE) { + throw new IllegalArgumentException( + String.format( + "%s is not supported on a %s: its read does not apply row filters or " + + "column masks.", + CoreOptions.QUERY_AUTH_ENABLED.key(), tableType)); + } for (DataField field : schema.fields()) { validateDefaultValue(field.type(), field.defaultValue()); } diff --git a/paimon-core/src/main/java/org/apache/paimon/catalog/TableQueryAuthResult.java b/paimon-core/src/main/java/org/apache/paimon/catalog/TableQueryAuthResult.java index 0f3a55ac832e..d29853509787 100644 --- a/paimon-core/src/main/java/org/apache/paimon/catalog/TableQueryAuthResult.java +++ b/paimon-core/src/main/java/org/apache/paimon/catalog/TableQueryAuthResult.java @@ -25,6 +25,7 @@ import org.apache.paimon.predicate.FieldRef; import org.apache.paimon.predicate.LeafPredicate; import org.apache.paimon.predicate.Predicate; +import org.apache.paimon.predicate.PredicateBuilder; import org.apache.paimon.predicate.PredicateVisitor; import org.apache.paimon.predicate.Transform; import org.apache.paimon.reader.RecordReader; @@ -45,6 +46,7 @@ import java.io.Serializable; import java.util.ArrayDeque; import java.util.ArrayList; +import java.util.Collections; import java.util.Deque; import java.util.HashMap; import java.util.HashSet; @@ -65,7 +67,8 @@ public class TableQueryAuthResult implements Serializable { private final @Nullable List filter; private final @Nullable Map columnMasking; - // lazily parsed views of the JSON rules; transient so serialization stays unchanged + // Lazily parsed views of the JSON rules; transient so serialization stays unchanged. No + // invalidation needed: an instance is immutable and rebuilt for every plan(). private transient volatile Optional parsedFilter; private transient volatile Map parsedMasking; @@ -100,6 +103,60 @@ public RowType widenReadType(RowType tableType, RowType readType) { tableType, readType, requiredAuthFields(readType.getFieldNames())); } + /** + * Drops the conjuncts of {@code predicate} referencing any of {@code fields}; returns null when + * nothing remains. Used to keep raw-statistics pushdown off masked columns. + */ + @Nullable + public static Predicate excludeFields(Predicate predicate, Set fields) { + return filterConjuncts(predicate, fields, true); + } + + /** + * Keeps only the conjuncts of {@code predicate} referencing any of {@code fields}; returns null + * when none does. + */ + @Nullable + public static Predicate retainFields(Predicate predicate, Set fields) { + return filterConjuncts(predicate, fields, false); + } + + @Nullable + private static Predicate filterConjuncts( + Predicate predicate, Set fields, boolean keepDisjoint) { + List kept = new ArrayList<>(); + for (Predicate conjunct : PredicateBuilder.splitAnd(predicate)) { + if (Collections.disjoint(PredicateVisitor.collectFieldNames(conjunct), fields) + == keepDisjoint) { + kept.add(conjunct); + } + } + if (kept.isEmpty()) { + return null; + } + return kept.size() == 1 ? kept.get(0) : PredicateBuilder.and(kept); + } + + /** + * Every column read by the conjuncts of {@code filter} that touch {@code maskTargets}. Their + * unmasked operands count too, since splitAnd does not split a disjunction. + */ + public static Set postMaskFilterFields( + @Nullable Predicate filter, Set maskTargets) { + if (filter == null || maskTargets.isEmpty()) { + return Collections.emptySet(); + } + Set maskedInFilter = new HashSet<>(PredicateVisitor.collectFieldNames(filter)); + maskedInFilter.retainAll(maskTargets); + if (maskedInFilter.isEmpty()) { + return Collections.emptySet(); + } + Predicate retained = retainFields(filter, maskedInFilter); + return retained == null + ? Collections.emptySet() + : new HashSet<>(PredicateVisitor.collectFieldNames(retained)); + } + /** Appends the missing {@code ruleFields} of {@code tableType} to {@code readType}. */ @Nullable public static RowType appendMissingFields( @@ -189,25 +246,38 @@ private Map parseColumnMasking() { result.put(column, transform); } } - return result; + // the cache is shared by every caller, so hand out a view that cannot rewrite the rules + return Collections.unmodifiableMap(result); } /** * Validates that every column the auth rules reference exists in the table's latest - * schema (not a time-travel-pinned one). A rule keyed by a since-renamed column looks just like - * an unprojected one at read time and would silently stop masking: fail closed instead. + * schema. A rule keyed by a since-renamed column would silently stop masking; fail closed. */ public void validateAgainstSchema(RowType tableType, @Nullable List projectedFields) { - for (Map.Entry entry : extractColumnMasking().entrySet()) { + Map masking = extractColumnMasking(); + for (Map.Entry entry : masking.entrySet()) { String target = entry.getKey(); - // a mask on an unprojected system column is inert (never in the output); don't reject + // a mask on an unprojected system column never reaches the output; inert, don't reject if (SpecialFields.SYSTEM_FIELD_NAMES.contains(target) && (projectedFields == null || !projectedFields.contains(target))) { continue; } checkFieldExists("Column masking", target, tableType, projectedFields); - for (String input : PredicateVisitor.collectFieldNames(entry.getValue())) { + for (String input : PredicateVisitor.collectTransformFieldNames(entry.getValue())) { checkFieldExists("Column masking", input, tableType, projectedFields); + // A transform reads the raw row, so an input that is itself masked would be + // consumed unmasked and its raw value published through this target. Masking + // the target of another mask is only self-consistent if composed, which the + // read does not do; refuse the pair rather than leak. + if (!input.equals(target) && masking.containsKey(input)) { + throw new IllegalArgumentException( + String.format( + "Column masking on '%s' reads column '%s', which is masked " + + "too. The mask would be computed from the raw value " + + "of '%s' and expose it through '%s'.", + target, input, input, target)); + } } } for (String operand : PredicateVisitor.collectFieldNames(extractPredicate())) { @@ -216,42 +286,59 @@ public void validateAgainstSchema(RowType tableType, @Nullable List proj } /** - * Fails closed when a masked column is present in the read schema under a different name than - * the rule uses (renamed between the read snapshot and latest): name-based enforcement would - * skip the mask and leak the raw value. A column absent from the read schema is left inert. + * Fails closed when the read schema exposes a rule's column under a different name, where + * enforcing by name would skip it. A column absent from that schema stays inert. */ public void validateReadableWithoutRename(RowType latestType, RowType readType) { for (Map.Entry entry : extractColumnMasking().entrySet()) { - checkNotRenamed(entry.getKey(), latestType, readType); + checkNotRenamed("Column masking", entry.getKey(), latestType, readType); // a mask whose target is absent from the read schema is inert; skip its inputs if (readType.containsField(entry.getKey())) { - for (String input : PredicateVisitor.collectFieldNames(entry.getValue())) { - checkNotRenamed(input, latestType, readType); + for (String input : PredicateVisitor.collectTransformFieldNames(entry.getValue())) { + checkNotRenamed("Column masking", input, latestType, readType); } } } + // the row filter is remapped by name as well, so it needs the same binding check + for (String operand : PredicateVisitor.collectFieldNames(extractPredicate())) { + checkNotRenamed("Row filter", operand, latestType, readType); + } } - private static void checkNotRenamed(String field, RowType latestType, RowType readType) { - // present by name: enforced fine. absent from latest: already thrown. else: renamed? - if (readType.containsField(field) || !latestType.containsField(field)) { + private static void checkNotRenamed( + String rule, String field, RowType latestType, RowType readType) { + // absent from latest: already thrown by validateAgainstSchema + if (!latestType.containsField(field)) { return; } int id = latestType.getField(field).id(); + if (readType.containsField(field)) { + // a dropped and re-added column keeps the name but gets a fresh id, so the same + // name may be an unrelated column in the snapshot being read + if (readType.getField(field).id() != id) { + throw new IllegalArgumentException( + String.format( + "%s references column '%s' which the snapshot being read exposes " + + "as a different column of the same name (dropped and " + + "re-added since); refusing to read to avoid applying the " + + "rule to unrelated data.", + rule, field)); + } + return; + } if (readType.containsField(id)) { throw new IllegalArgumentException( String.format( - "Column masking references column '%s' which the snapshot being read " - + "exposes as '%s' (renamed since); refusing to read to avoid " - + "applying the rule by a stale name.", - field, readType.getField(id).name())); + "%s references column '%s' which the snapshot being read exposes as " + + "'%s' (renamed since); refusing to read to avoid applying " + + "the rule by a stale name.", + rule, field, readType.getField(id).name())); } } /** * The field names the auth rules read for a query projecting {@code projectedFields}: the - * row-filter operands, plus (transitively) the inputs of every mask whose target is readable — - * projected, or itself pulled in by the filter or another mask. + * row-filter operands, plus transitively the inputs of every mask whose target is readable. */ public Set requiredAuthFields(List projectedFields) { Map masking = extractColumnMasking(); @@ -269,7 +356,7 @@ public Set requiredAuthFields(List projectedFields) { if (mask == null) { continue; } - for (String input : PredicateVisitor.collectFieldNames(mask)) { + for (String input : PredicateVisitor.collectTransformFieldNames(mask)) { ruleFields.add(input); if (readable.add(input)) { newlyReadable.add(input); @@ -281,8 +368,7 @@ public Set requiredAuthFields(List projectedFields) { private static void checkFieldExists( String rule, String field, RowType tableType, @Nullable List projectedFields) { - // system fields (e.g. _ROW_ID) are readable metadata absent from the table schema, - // but only when the query actually projects them -- they cannot be widened in + // system fields (e.g. _ROW_ID) cannot be widened in; only usable when projected if (SpecialFields.SYSTEM_FIELD_NAMES.contains(field)) { if (projectedFields != null && projectedFields.contains(field)) { return; diff --git a/paimon-core/src/main/java/org/apache/paimon/globalindex/DataEvolutionBatchScan.java b/paimon-core/src/main/java/org/apache/paimon/globalindex/DataEvolutionBatchScan.java index 8929ef4c7bb6..b488c73664b0 100644 --- a/paimon-core/src/main/java/org/apache/paimon/globalindex/DataEvolutionBatchScan.java +++ b/paimon-core/src/main/java/org/apache/paimon/globalindex/DataEvolutionBatchScan.java @@ -96,6 +96,12 @@ public InnerTableScan withFilter(Predicate predicate) { } this.filter = predicate; + if (queryAuthEnabled()) { + // let the wrapped scan defer the filter: a conjunct on a masked column must not + // reach raw statistics or the global index + batchScan.withFilter(predicate); + return this; + } batchScan.snapshotReader().withFilter(predicate, rowIdSafeResidualFilter(predicate)); return this; } @@ -291,6 +297,11 @@ public Plan plan() { return wrapToIndexSplits(splits, rowRangeIndex, scoreGetter); } + private boolean queryAuthEnabled() { + CoreOptions options = table == null ? null : table.coreOptions(); + return options != null && options.queryAuthEnabled(); + } + private Optional evalGlobalIndex() { if (this.globalIndexResult != null) { return Optional.of(globalIndexResult); @@ -298,6 +309,10 @@ private Optional evalGlobalIndex() { if (filter == null) { return Optional.empty(); } + if (queryAuthEnabled()) { + // the index ranks raw values, which a mask may invalidate; fall back to a full scan + return Optional.empty(); + } CoreOptions options = table.coreOptions(); if (!options.globalIndexEnabled()) { return Optional.empty(); diff --git a/paimon-core/src/main/java/org/apache/paimon/operation/DataEvolutionFileStoreScan.java b/paimon-core/src/main/java/org/apache/paimon/operation/DataEvolutionFileStoreScan.java index 5622d995d356..22f70f0cbaed 100644 --- a/paimon-core/src/main/java/org/apache/paimon/operation/DataEvolutionFileStoreScan.java +++ b/paimon-core/src/main/java/org/apache/paimon/operation/DataEvolutionFileStoreScan.java @@ -127,8 +127,8 @@ public DataEvolutionFileStoreScan withFilter(Predicate predicate) { @Override public FileStoreScan withReadType(RowType readType) { - // a type without user columns does not prune column files; reset, as this - // method may be called again + // a type without user columns prunes nothing; assign unconditionally, this method + // may be recalled if (readType != null && readType.getFields().stream() .anyMatch(f -> !SpecialFields.isSystemField(f.id()))) { diff --git a/paimon-core/src/main/java/org/apache/paimon/table/query/LocalTableQuery.java b/paimon-core/src/main/java/org/apache/paimon/table/query/LocalTableQuery.java index f189d886823f..6f355d6602e5 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/query/LocalTableQuery.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/query/LocalTableQuery.java @@ -94,6 +94,12 @@ public class LocalTableQuery implements TableQuery { public LocalTableQuery(FileStoreTable table) { this.options = table.coreOptions(); + if (options.queryAuthEnabled()) { + // the lookup cache serves rows straight from the store, past the auth reader + throw new UnsupportedOperationException( + "Local table query is not supported on a query-auth table: it returns raw " + + "values, outside row filters and column masks."); + } this.tableView = new ConcurrentHashMap<>(); FileStore tableStore = table.store(); if (!(tableStore instanceof KeyValueFileStore)) { diff --git a/paimon-core/src/main/java/org/apache/paimon/table/source/AbstractBatchTableScan.java b/paimon-core/src/main/java/org/apache/paimon/table/source/AbstractBatchTableScan.java index c41933221be6..b4d927819a4e 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/source/AbstractBatchTableScan.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/source/AbstractBatchTableScan.java @@ -19,6 +19,7 @@ package org.apache.paimon.table.source; import org.apache.paimon.CoreOptions; +import org.apache.paimon.catalog.TableQueryAuthResult; import org.apache.paimon.manifest.PartitionEntry; import org.apache.paimon.predicate.Predicate; import org.apache.paimon.predicate.SortValue; @@ -151,6 +152,15 @@ protected Plan postProcessPlan(Plan plan) { @Override public List listPartitionEntries() { + // partition listing bypasses plan(), so resolve the masks here too: pushing a filter on a + // masked column against raw partition values would drop partitions the query matches + TableQueryAuthResult authResult = authQuery(); + this.authMaskedFields = + authResult == null + ? java.util.Collections.emptySet() + : authResult.extractColumnMasking().keySet(); + rejectMaskedPartitionFilter(); + ensureFilterPushdown(authMaskedFields); if (startingScanner == null) { startingScanner = createStartingScanner(false); } @@ -222,6 +232,10 @@ private Optional applyPushDownTopN() { } SortValue order = orders.get(0); + if (authMaskedFields.contains(order.field().name())) { + // the pruning below reads raw statistics; a mask may alter the ordering column + return Optional.empty(); + } DataType type = order.field().type(); if (!minmaxAvailable(type)) { return Optional.empty(); diff --git a/paimon-core/src/main/java/org/apache/paimon/table/source/AbstractDataTableRead.java b/paimon-core/src/main/java/org/apache/paimon/table/source/AbstractDataTableRead.java index 9c20a7f8a2a7..64c5ea4a6be3 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/source/AbstractDataTableRead.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/source/AbstractDataTableRead.java @@ -24,6 +24,7 @@ import org.apache.paimon.disk.IOManager; import org.apache.paimon.predicate.Predicate; import org.apache.paimon.predicate.PredicateProjectionConverter; +import org.apache.paimon.predicate.PredicateVisitor; import org.apache.paimon.predicate.Transform; import org.apache.paimon.reader.RecordReader; import org.apache.paimon.schema.TableSchema; @@ -34,6 +35,7 @@ import javax.annotation.Nullable; import java.io.IOException; +import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; @@ -50,8 +52,12 @@ public abstract class AbstractDataTableRead implements InnerTableRead { private Predicate predicate; private final TableSchema schema; - // The read type the subclass reads with; differs from readType only when widened for - // auth, and is fixed once a reader exists (split reads cache their format readers). + // reader-level filtering sees raw values, so it stays off for auth-enabled tables, + // as read-level TopN already does (see ReadBuilderImpl) + private final boolean queryAuthEnabled; + + // the read type the subclass reads with; widened for auth when needed, and fixed + // once a reader exists (split reads cache their format readers) @Nullable private RowType appliedReadType; // blob-view columns that only resolve through the dedicated blob-view read path @@ -60,13 +66,16 @@ public abstract class AbstractDataTableRead implements InnerTableRead { public AbstractDataTableRead(@Nullable TableSchema schema) { this.schema = schema; Set blobViewFields = Collections.emptySet(); + boolean queryAuthEnabled = false; if (schema != null) { CoreOptions options = CoreOptions.fromMap(schema.options()); if (options.blobViewResolveEnabled()) { blobViewFields = options.blobViewField(); } + queryAuthEnabled = options.queryAuthEnabled(); } this.resolvedBlobViewFields = blobViewFields; + this.queryAuthEnabled = queryAuthEnabled; } public abstract void applyReadType(RowType readType); @@ -81,6 +90,9 @@ public TableRead withIOManager(IOManager ioManager) { @Override public final InnerTableRead withFilter(Predicate predicate) { this.predicate = predicate; + if (queryAuthEnabled) { + return this; + } return innerWithFilter(predicate); } @@ -163,7 +175,24 @@ private RecordReader readSplit(Split split) throws IOException { private RecordReader authedReader(Split split, TableQueryAuthResult authResult) throws IOException { List readFields = currentReadType().getFieldNames(); - Set ruleFields = authResult.requiredAuthFields(readFields); + // masked filter columns are read and masked like rule fields, then evaluated post-mask + Set maskedFilterFields = + maskedFilterFields(authResult.extractColumnMasking().keySet()); + // a retained conjunct may also reference unmasked columns; all must be readable + Set postMaskFilterFields = + TableQueryAuthResult.postMaskFilterFields( + predicate, authResult.extractColumnMasking().keySet()); + List visibleFields = readFields; + if (!postMaskFilterFields.isEmpty()) { + visibleFields = new ArrayList<>(readFields); + for (String field : postMaskFilterFields) { + if (!visibleFields.contains(field)) { + visibleFields.add(field); + } + } + } + Set ruleFields = authResult.requiredAuthFields(visibleFields); + ruleFields.addAll(postMaskFilterFields); RowType widened = widenedReadType(authResult, ruleFields); if (widened != null && !widened.equals(appliedReadType)) { applyReadType(widened); @@ -205,9 +234,47 @@ private RecordReader authedReader(Split split, TableQueryAuthResult outputType, authResult.extractPredicate(), selectedMasking); + reader = filterMaskedConjuncts(reader, outputType, maskedFilterFields); return backProject(reader); } + /** The columns of the query filter that the current auth rules mask. */ + private Set maskedFilterFields(Set maskTargets) { + if (predicate == null || maskTargets.isEmpty()) { + return Collections.emptySet(); + } + Set fields = new HashSet<>(PredicateVisitor.collectFieldNames(predicate)); + fields.retainAll(maskTargets); + return fields; + } + + /** + * Evaluates the filter conjuncts on masked columns, on the masked output: they are never pushed + * down, and engines do not re-evaluate the conjuncts they consumed. + */ + private RecordReader filterMaskedConjuncts( + RecordReader reader, RowType outputType, Set maskedFilterFields) { + if (maskedFilterFields.isEmpty()) { + return reader; + } + Predicate maskedPart = TableQueryAuthResult.retainFields(predicate, maskedFilterFields); + if (maskedPart == null) { + return reader; + } + int[] projection = schema.logicalRowType().getFieldIndices(outputType.getFieldNames()); + Optional remapped = + maskedPart.visit(PredicateProjectionConverter.fromProjection(projection)); + if (!remapped.isPresent()) { + throw new IllegalStateException( + "Filter on masked columns " + + maskedFilterFields + + " cannot be evaluated on read schema " + + outputType.getFieldNames()); + } + Predicate filter = remapped.get(); + return reader.filter(filter::test); + } + /** * Project auth-widened rows back to the query's read type — on every split, since the widened * read schema stays in effect even for splits without auth rules. @@ -223,9 +290,8 @@ private RecordReader backProject(RecordReader reader) } /** - * The read type widened with the unprojected columns the auth rules read (mirrors the - * row-filter augmentation of #8447), or null when the projection already covers them. The - * projected fields are kept as-is to preserve nested pruning. + * The read type widened with the unprojected columns the auth rules read, or null when the + * projection already covers them. Projected fields are kept as-is to preserve nested pruning. */ @Nullable private RowType widenedReadType(TableQueryAuthResult authResult, Set ruleFields) { diff --git a/paimon-core/src/main/java/org/apache/paimon/table/source/AbstractDataTableScan.java b/paimon-core/src/main/java/org/apache/paimon/table/source/AbstractDataTableScan.java index 986e9de12da1..a86f19b014c2 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/source/AbstractDataTableScan.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/source/AbstractDataTableScan.java @@ -30,6 +30,7 @@ import org.apache.paimon.options.Options; import org.apache.paimon.partition.PartitionPredicate; import org.apache.paimon.predicate.Predicate; +import org.apache.paimon.predicate.PredicateVisitor; import org.apache.paimon.schema.SchemaManager; import org.apache.paimon.schema.TableSchema; import org.apache.paimon.table.source.snapshot.CompactedStartingScanner; @@ -67,11 +68,14 @@ import javax.annotation.Nullable; +import java.util.ArrayList; +import java.util.Collections; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Optional; +import java.util.Set; import java.util.TimeZone; import static org.apache.paimon.CoreOptions.FULL_COMPACTION_DELTA_COMMITS; @@ -96,6 +100,15 @@ abstract class AbstractDataTableScan implements DataTableScan { // Whether the auth predicate has a non-partition part (enforced only at read time). Used by // AbstractBatchTableScan to disable limit push down; not pushed through withFilter. protected boolean authHasNonPartitionFilter; + // auth state, refreshed each plan(). The filter is pushed once, without the conjuncts on + // masked columns, whose raw statistics a mask invalidates; the partition fields are + // replaced on each push, as ManifestsReader#withPartitionFilter overwrites rather than ands. + protected Set authMaskedFields = Collections.emptySet(); + @Nullable private RowType appliedScanReadType; + @Nullable private Predicate userFilter; + private boolean filterPushed = false; + private Set pushedMaskedFields = Collections.emptySet(); + private Set partitionFilterFields = Collections.emptySet(); protected AbstractDataTableScan( TableSchema schema, @@ -110,14 +123,17 @@ protected AbstractDataTableScan( this.queryAuth = queryAuth; } - // the read type last pushed to the snapshot reader; widened for auth when needed - @Nullable private RowType appliedScanReadType; - @Override public final TableScan.Plan plan() { TableQueryAuthResult queryAuthResult = authQuery(); // Always apply/clear the auth filter so removing auth leaves no stale partition pruning. applyAuthFilter(queryAuthResult == null ? null : queryAuthResult.extractPredicate()); + this.authMaskedFields = + queryAuthResult == null + ? Collections.emptySet() + : queryAuthResult.extractColumnMasking().keySet(); + rejectMaskedPartitionFilter(); + ensureFilterPushdown(authMaskedFields); applyAuthReadType(queryAuthResult); Plan plan = planWithoutAuth(); if (queryAuthResult != null) { @@ -161,7 +177,13 @@ private void applyAuthFilter(@Nullable Predicate authPredicate) { @Override public InnerTableScan withFilter(Predicate predicate) { - snapshotReader.withFilter(predicate); + if (!options.queryAuthEnabled()) { + // no masks to strip; push now, else chain-table sub-scans read before plan() + snapshotReader.withFilter(predicate); + return this; + } + // deferred to plan(), which strips conjuncts on masked columns before stats pruning + this.userFilter = predicate; return this; } @@ -187,30 +209,50 @@ public InnerTableScan withReadType(@Nullable RowType readType) { @Override public AbstractDataTableScan withPartitionFilter(Map partitionSpec) { + partitionFilterFields = + partitionSpec == null + ? Collections.emptySet() + : new HashSet<>(partitionSpec.keySet()); snapshotReader.withPartitionFilter(partitionSpec); return this; } @Override public AbstractDataTableScan withPartitionFilter(List partitions) { + // binary partitions carry no field names; assume every partition key + partitionFilterFields = + partitions == null ? Collections.emptySet() : new HashSet<>(schema.partitionKeys()); snapshotReader.withPartitionFilter(partitions); return this; } @Override public AbstractDataTableScan withPartitionsFilter(List> partitions) { + Set fields = new HashSet<>(); + if (partitions != null) { + partitions.forEach(spec -> fields.addAll(spec.keySet())); + } + partitionFilterFields = fields; snapshotReader.withPartitionsFilter(partitions); return this; } @Override public AbstractDataTableScan withPartitionFilter(PartitionPredicate partitionPredicate) { + partitionFilterFields = + partitionPredicate == null + ? Collections.emptySet() + : partitionPredicateFields(partitionPredicate); snapshotReader.withPartitionFilter(partitionPredicate); return this; } @Override public InnerTableScan withPartitionFilter(Predicate predicate) { + partitionFilterFields = + predicate == null + ? Collections.emptySet() + : PredicateVisitor.collectFieldNames(predicate); snapshotReader.withPartitionFilter(predicate); return this; } @@ -234,9 +276,7 @@ protected TableQueryAuthResult authQuery() { List select = readType == null ? null : readType.getFieldNames(); TableQueryAuthResult result = queryAuth.auth(select); if (result != null && result.hasRules()) { - // validated on every plan (this path already pays an auth-service call per plan), so - // schema changes under a live scan fail closed: references stale in the latest schema, - // or renamed relative to the schema being read (e.g. time travel), are rejected + // re-validated every plan, so a schema change under a live scan fails closed RowType latestSchema = schemaManager .latest() @@ -266,6 +306,67 @@ public InnerTableScan withRowRangeIndex(RowRangeIndex rowRangeIndex) { return this; } + /** The partition columns a predicate references, or all of them when it cannot be read. */ + private Set partitionPredicateFields(PartitionPredicate partitionPredicate) { + if (partitionPredicate instanceof PartitionPredicate.DefaultPartitionPredicate) { + return PredicateVisitor.collectFieldNames( + ((PartitionPredicate.DefaultPartitionPredicate) partitionPredicate) + .predicate()); + } + return new HashSet<>(schema.partitionKeys()); + } + + /** + * A partition predicate is consumed by pruning and, in Spark, dropped from post-scan + * evaluation, so it can never be re-checked on the masked value. Fail closed. One routed + * through withFilter is fine: that path defers it and evaluates it post-mask. + */ + protected void rejectMaskedPartitionFilter() { + if (partitionFilterFields.isEmpty() || authMaskedFields.isEmpty()) { + return; + } + for (String partitionKey : partitionFilterFields) { + if (authMaskedFields.contains(partitionKey)) { + throw new UnsupportedOperationException( + String.format( + "A partition filter cannot be enforced on masked partition key " + + "'%s': engines push partition predicates past the " + + "reader, so it would be matched against the raw " + + "partition value.", + partitionKey)); + } + } + } + + /** + * Pushes the query filter once, minus the conjuncts on masked columns. Also called by partition + * listing; a mask found later on an already-pushed column fails closed. + */ + protected final void ensureFilterPushdown(Set maskedFields) { + if (userFilter == null) { + return; + } + Set maskedInFilter = new HashSet<>(maskedFields); + maskedInFilter.retainAll(PredicateVisitor.collectFieldNames(userFilter)); + if (!maskedInFilter.isEmpty()) { + // masked conjuncts drop rows at read time only: keep limit/TopN pruning off + authHasNonPartitionFilter = true; + } + if (!filterPushed) { + Predicate effective = + maskedInFilter.isEmpty() + ? userFilter + : TableQueryAuthResult.excludeFields(userFilter, maskedInFilter); + snapshotReader.withFilter(userFilter, effective); + filterPushed = true; + pushedMaskedFields = maskedInFilter; + } else if (!pushedMaskedFields.containsAll(maskedInFilter)) { + throw new IllegalStateException( + "Query auth rules changed and now mask a pushed-down filter column. " + + "Recreate the scan to apply the new rules."); + } + } + /** * Push the auth-widened read type to the snapshot reader before planning, so file-level column * pruning keeps the files of the columns the rules read. @@ -276,7 +377,25 @@ private void applyAuthReadType(@Nullable TableQueryAuthResult queryAuthResult) { } RowType desired = readType; if (queryAuthResult != null && queryAuthResult.hasRules()) { - RowType widened = queryAuthResult.widenReadType(schema.logicalRowType(), readType); + // post-mask conjuncts are evaluated at read time; their columns must survive planning + List seed = readType.getFieldNames(); + Set postMask = + TableQueryAuthResult.postMaskFilterFields( + userFilter, queryAuthResult.extractColumnMasking().keySet()); + if (!postMask.isEmpty()) { + seed = new ArrayList<>(seed); + for (String field : postMask) { + if (!seed.contains(field)) { + seed.add(field); + } + } + } + Set ruleFields = queryAuthResult.requiredAuthFields(seed); + // requiredAuthFields returns what the rules read, not the seed itself + ruleFields.addAll(postMask); + RowType widened = + TableQueryAuthResult.appendMissingFields( + schema.logicalRowType(), readType, ruleFields); if (widened != null) { desired = widened; } diff --git a/paimon-core/src/main/java/org/apache/paimon/table/source/BatchVectorSearchBuilderImpl.java b/paimon-core/src/main/java/org/apache/paimon/table/source/BatchVectorSearchBuilderImpl.java index 81916cac7bcf..f47a8959df32 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/source/BatchVectorSearchBuilderImpl.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/source/BatchVectorSearchBuilderImpl.java @@ -125,6 +125,7 @@ public BatchVectorSearchBuilder withOption(String key, String value) { @Override public VectorScan newVectorScan() { + rejectUnderQueryAuth(); if (isPrimaryKeyVectorSearch()) { return new PrimaryKeyVectorScan( table, @@ -157,4 +158,12 @@ protected boolean isPrimaryKeyVectorSearch() { return vectorColumn != null && table.coreOptions().primaryKeyVectorIndexColumns().contains(vectorColumn.name()); } + + private void rejectUnderQueryAuth() { + if (table.coreOptions().queryAuthEnabled()) { + throw new UnsupportedOperationException( + "Search is not supported on a query-auth table: the index ranks raw values, " + + "which a column mask invalidates."); + } + } } diff --git a/paimon-core/src/main/java/org/apache/paimon/table/source/DataTableStreamScan.java b/paimon-core/src/main/java/org/apache/paimon/table/source/DataTableStreamScan.java index a7c6bf0c8890..02806cfe33fa 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/source/DataTableStreamScan.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/source/DataTableStreamScan.java @@ -106,7 +106,6 @@ public DataTableStreamScan( @Override public DataTableStreamScan withFilter(Predicate predicate) { super.withFilter(predicate); - snapshotReader.withFilter(predicate); return this; } diff --git a/paimon-core/src/main/java/org/apache/paimon/table/source/FullTextSearchBuilderImpl.java b/paimon-core/src/main/java/org/apache/paimon/table/source/FullTextSearchBuilderImpl.java index dccd3d7da20f..a692aec1eed0 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/source/FullTextSearchBuilderImpl.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/source/FullTextSearchBuilderImpl.java @@ -72,6 +72,7 @@ public FullTextSearchBuilder withQuery(String fieldName, String query) { @Override public FullTextScan newFullTextScan() { + rejectUnderQueryAuth(); DataField textColumn = textColumn(); Optional definition = primaryKeyFullTextDefinition(textColumn); return definition.isPresent() @@ -128,4 +129,12 @@ FullTextSearchBuilderImpl withSnapshot(Snapshot snapshot) { this.pinnedSnapshot = snapshot; return this; } + + private void rejectUnderQueryAuth() { + if (table.coreOptions().queryAuthEnabled()) { + throw new UnsupportedOperationException( + "Search is not supported on a query-auth table: the index ranks raw values, " + + "which a column mask invalidates."); + } + } } diff --git a/paimon-core/src/main/java/org/apache/paimon/table/source/HybridSearchBuilderImpl.java b/paimon-core/src/main/java/org/apache/paimon/table/source/HybridSearchBuilderImpl.java index 7d3725444d7d..ac9b965c2e5b 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/source/HybridSearchBuilderImpl.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/source/HybridSearchBuilderImpl.java @@ -128,6 +128,7 @@ public HybridSearchBuilder withWeightedScoreRanker() { @Override public List routeBuilders() { + rejectUnderQueryAuth(); validateSearch(); Snapshot snapshot = null; @@ -377,4 +378,13 @@ protected FullTextSearchBuilder newFullTextSearchBuilder(HybridSearchRoute route } return fullTextSearchBuilder; } + + private void rejectUnderQueryAuth() { + if (table instanceof FileStoreTable + && ((FileStoreTable) table).coreOptions().queryAuthEnabled()) { + throw new UnsupportedOperationException( + "Search is not supported on a query-auth table: the index ranks raw values, " + + "which a column mask invalidates."); + } + } } diff --git a/paimon-core/src/main/java/org/apache/paimon/table/source/PrimaryKeyBatchScan.java b/paimon-core/src/main/java/org/apache/paimon/table/source/PrimaryKeyBatchScan.java index e50ff3772d29..b50f5dfcb5c6 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/source/PrimaryKeyBatchScan.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/source/PrimaryKeyBatchScan.java @@ -19,6 +19,7 @@ package org.apache.paimon.table.source; import org.apache.paimon.Snapshot; +import org.apache.paimon.catalog.TableQueryAuthResult; import org.apache.paimon.globalindex.GlobalIndexResult; import org.apache.paimon.index.GlobalIndexMeta; import org.apache.paimon.index.IndexFileHandler; @@ -135,8 +136,15 @@ protected Plan postProcessPlan(Plan dataPlan) { || snapshotPlan.splits().isEmpty()) { return dataPlan; } + // the index is built over raw values, so a conjunct on a masked column must not reach it + Predicate indexCandidate = + filter == null || authMaskedFields.isEmpty() + ? filter + : TableQueryAuthResult.excludeFields(filter, authMaskedFields); Predicate indexFilter = - filter == null ? null : filter.visit(indexPredicateExtractor).orElse(null); + indexCandidate == null + ? null + : indexCandidate.visit(indexPredicateExtractor).orElse(null); if (indexFilter == null) { return dataPlan; } diff --git a/paimon-core/src/main/java/org/apache/paimon/table/source/ReadBuilderImpl.java b/paimon-core/src/main/java/org/apache/paimon/table/source/ReadBuilderImpl.java index 9d2be281b2c7..ff088731ff6c 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/source/ReadBuilderImpl.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/source/ReadBuilderImpl.java @@ -245,10 +245,11 @@ public TableRead newRead() { read.withReadType(readType); } if (queryAuthEnabled) { - // Skip TopN (engine re-applies it); apply the limit after auth only without a TopN, - // else an unordered limit could drop sorted rows. + // Skip TopN and, with a filter or a TopN present, the limit as well: the engine + // re-applies them. The reader does not evaluate the query filter on an auth-enabled + // table, so capping the rows here would cut away the rows that actually match. if (topN == null && limit != null) { - return new LimitTableRead(read, limit); + return new LimitTableRead(read, limit, filter != null); } return read; } @@ -291,10 +292,15 @@ private static class LimitTableRead implements TableRead { private final TableRead delegate; private final int limit; + // with a filter the reader only evaluates it once executeFilter() is requested; + // otherwise the engine does, after this limit, so capping here would drop matches + private final boolean filterPresent; + private boolean filterExecutedByReader = false; - private LimitTableRead(TableRead delegate, int limit) { + private LimitTableRead(TableRead delegate, int limit, boolean filterPresent) { this.delegate = delegate; this.limit = limit; + this.filterPresent = filterPresent; } @Override @@ -306,6 +312,7 @@ public TableRead withMetricRegistry(MetricRegistry registry) { @Override public TableRead executeFilter() { delegate.executeFilter(); + this.filterExecutedByReader = true; return this; } @@ -338,6 +345,9 @@ public RecordReader createReader(TableScan.Plan plan) throws IOExce } private RecordReader limit(RecordReader reader) { + if (filterPresent && !filterExecutedByReader) { + return reader; + } // Stop reading once the limit is reached (return EOF), rather than filtering and // draining the rest of the data. return new RecordReader() { diff --git a/paimon-core/src/main/java/org/apache/paimon/table/source/VectorSearchBuilderImpl.java b/paimon-core/src/main/java/org/apache/paimon/table/source/VectorSearchBuilderImpl.java index d5c2526a72c0..0bda59da80ac 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/source/VectorSearchBuilderImpl.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/source/VectorSearchBuilderImpl.java @@ -128,6 +128,7 @@ public VectorSearchBuilder withOption(String key, String value) { @Override public VectorScan newVectorScan() { + rejectUnderQueryAuth(); if (isPrimaryKeyVectorSearch()) { return new PrimaryKeyVectorScan( table, @@ -160,4 +161,12 @@ public VectorSearchBuilderImpl withSnapshot(Snapshot snapshot) { this.pinnedSnapshot = snapshot; return this; } + + private void rejectUnderQueryAuth() { + if (table.coreOptions().queryAuthEnabled()) { + throw new UnsupportedOperationException( + "Search is not supported on a query-auth table: the index ranks raw values, " + + "which a column mask invalidates."); + } + } } diff --git a/paimon-core/src/main/java/org/apache/paimon/table/system/SystemTableLoader.java b/paimon-core/src/main/java/org/apache/paimon/table/system/SystemTableLoader.java index 3bc083f3e24f..615e078aa1ba 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/system/SystemTableLoader.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/system/SystemTableLoader.java @@ -94,9 +94,25 @@ public class SystemTableLoader { public static final List GLOBAL_SYSTEM_TABLES = Arrays.asList(ALL_TABLES, ALL_PARTITIONS, ALL_TABLE_OPTIONS, CATALOG_OPTIONS); + /** + * System tables built from raw DataSplit metadata -- file names, row counts, per-column min/max + * -- none of which a column mask covers. + */ + private static final List PHYSICAL_METADATA_TABLES = + Arrays.asList(FILES, FILE_KEY_RANGES, BINLOG); + @Nullable public static Table load(String type, FileStoreTable dataTable) { - return Optional.ofNullable(SYSTEM_TABLE_LOADERS.get(type.toLowerCase())) + String name = type.toLowerCase(); + if (PHYSICAL_METADATA_TABLES.contains(name) && dataTable.coreOptions().queryAuthEnabled()) { + throw new UnsupportedOperationException( + String.format( + "System table '%s' is not supported on a query-auth table: it reports " + + "raw file statistics, which column masking cannot be applied " + + "to.", + name)); + } + return Optional.ofNullable(SYSTEM_TABLE_LOADERS.get(name)) .map(f -> f.apply(dataTable)) .orElse(null); } diff --git a/paimon-core/src/test/java/org/apache/paimon/catalog/TableQueryAuthResultTest.java b/paimon-core/src/test/java/org/apache/paimon/catalog/TableQueryAuthResultTest.java index ef5fb1726a25..4573ceae83e8 100644 --- a/paimon-core/src/test/java/org/apache/paimon/catalog/TableQueryAuthResultTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/catalog/TableQueryAuthResultTest.java @@ -20,7 +20,10 @@ import org.apache.paimon.data.BinaryString; import org.apache.paimon.predicate.ConcatWsTransform; +import org.apache.paimon.predicate.Equal; import org.apache.paimon.predicate.FieldRef; +import org.apache.paimon.predicate.FieldTransform; +import org.apache.paimon.predicate.LeafPredicate; import org.apache.paimon.predicate.Predicate; import org.apache.paimon.types.DataTypes; import org.apache.paimon.types.RowType; @@ -33,6 +36,7 @@ import java.util.Map; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatCode; import static org.assertj.core.api.Assertions.assertThatThrownBy; /** Tests that malformed query-authorization definitions cannot be silently ignored. */ @@ -97,6 +101,14 @@ void testInvalidColumnMaskFailsClosed() { new org.apache.paimon.types.DataField(0, "display", DataTypes.STRING()), new org.apache.paimon.types.DataField(1, "extra", DataTypes.STRING())); + private static String filterJson() { + return JsonSerdeUtil.toFlatJson( + LeafPredicate.of( + new FieldTransform(new FieldRef(1, "extra", DataTypes.STRING())), + Equal.INSTANCE, + Collections.singletonList(BinaryString.fromString("x")))); + } + private static String maskJson() { return JsonSerdeUtil.toFlatJson( new ConcatWsTransform( @@ -105,6 +117,45 @@ private static String maskJson() { new FieldRef(1, "extra", DataTypes.STRING())))); } + @Test + public void testValidateRejectsReAddedColumnOfSameName() { + Map masking = Collections.singletonMap("display", maskJson()); + TableQueryAuthResult result = new TableQueryAuthResult(null, masking); + + // same names and ids: the rule binds to the same physical columns + assertThatCode(() -> result.validateReadableWithoutRename(TABLE_TYPE, TABLE_TYPE)) + .doesNotThrowAnyException(); + + // the mask input 'extra' was dropped and re-added, so the latest schema gives it a fresh + // id. A time-travel read of the pre-drop snapshot still has an 'extra', but it is an + // unrelated column; resolving the rule by name would mask with its values. + RowType latest = + RowType.of( + new org.apache.paimon.types.DataField(0, "display", DataTypes.STRING()), + new org.apache.paimon.types.DataField(7, "extra", DataTypes.STRING())); + assertThatThrownBy(() -> result.validateReadableWithoutRename(latest, TABLE_TYPE)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("dropped and re-added"); + } + + @Test + public void testValidateRejectsReAddedColumnForRowFilter() { + // a row filter is remapped by name too, so it needs the same binding check as a mask + TableQueryAuthResult result = + new TableQueryAuthResult(Collections.singletonList(filterJson()), null); + assertThatCode(() -> result.validateReadableWithoutRename(TABLE_TYPE, TABLE_TYPE)) + .doesNotThrowAnyException(); + + RowType latest = + RowType.of( + new org.apache.paimon.types.DataField(0, "display", DataTypes.STRING()), + new org.apache.paimon.types.DataField(7, "extra", DataTypes.STRING())); + assertThatThrownBy(() -> result.validateReadableWithoutRename(latest, TABLE_TYPE)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("Row filter") + .hasMessageContaining("dropped and re-added"); + } + @Test public void testHasRules() { assertThat(new TableQueryAuthResult(null, null).hasRules()).isFalse(); diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTest.java b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTest.java index 684f80105f78..618185e3637b 100644 --- a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTest.java @@ -83,6 +83,7 @@ import org.apache.paimon.table.Table; import org.apache.paimon.table.TableSnapshot; import org.apache.paimon.table.object.ObjectTable; +import org.apache.paimon.table.query.LocalTableQuery; import org.apache.paimon.table.sink.BatchTableCommit; import org.apache.paimon.table.sink.BatchTableWrite; import org.apache.paimon.table.sink.BatchWriteBuilder; @@ -96,6 +97,7 @@ import org.apache.paimon.table.source.Split; import org.apache.paimon.table.source.StreamTableScan; import org.apache.paimon.table.source.TableRead; +import org.apache.paimon.table.source.TableScan; import org.apache.paimon.table.system.SystemTableLoader; import org.apache.paimon.types.DataField; import org.apache.paimon.types.DataTypes; @@ -3906,13 +3908,22 @@ void testColumnMaskingApplyOnRead() throws Exception { private Table createMaskingAuthTable( Identifier identifier, List fields, Map extraOptions) throws Exception { + return createMaskingAuthTable( + identifier, fields, Collections.emptyList(), Collections.emptyList(), extraOptions); + } + + private Table createMaskingAuthTable( + Identifier identifier, + List fields, + List partitionKeys, + List primaryKeys, + Map extraOptions) + throws Exception { catalog.createDatabase(identifier.getDatabaseName(), true); Map options = new HashMap<>(extraOptions); options.put(QUERY_AUTH_ENABLED.key(), "true"); catalog.createTable( - identifier, - new Schema(fields, Collections.emptyList(), Collections.emptyList(), options, ""), - true); + identifier, new Schema(fields, partitionKeys, primaryKeys, options, ""), true); return catalog.getTable(identifier); } @@ -4712,6 +4723,51 @@ void testColumnMaskingRejectsUnprojectedBlobViewInput() throws Exception { .hasMessageContaining("blob-view"); } + @Test + void testColumnMaskingDisablesTopNPushdown() throws Exception { + Identifier identifier = Identifier.create("test_table_db", "auth_table_masking_topn"); + List fields = new ArrayList<>(); + fields.add(new DataField(0, "display", DataTypes.INT())); + fields.add(new DataField(1, "source", DataTypes.INT())); + Table table = + createMaskingAuthTable( + identifier, + fields, + Collections.singletonMap( + CoreOptions.SOURCE_SPLIT_TARGET_SIZE.key(), "1 b")); + // two splits with opposite raw/masked ordering + for (int[] row : new int[][] {{100, 0}, {50, 1000}}) { + BatchWriteBuilder writeBuilder = table.newBatchWriteBuilder(); + BatchTableWrite write = writeBuilder.newWrite(); + write.write(GenericRow.of(row[0], row[1])); + BatchTableCommit commit = writeBuilder.newCommit(); + commit.commit(write.prepareCommit()); + write.close(); + commit.close(); + } + + // display := source inverts the ordering the raw statistics suggest + Map masking = new HashMap<>(); + masking.put("display", new FieldTransform(new FieldRef(1, "source", DataTypes.INT()))); + setColumnMasking(identifier, masking); + + TopN topN = + new TopN(new FieldRef(0, "display", DataTypes.INT()), DESCENDING, NULLS_LAST, 1); + ReadBuilder readBuilder = + table.newReadBuilder().withProjection(new int[] {0}).withTopN(topN); + List splits = readBuilder.newScan().plan().splits(); + List rows = + collectRows( + readBuilder.newRead().createReader(splits), + table.rowType().project("display")); + // split pruning must not drop the split holding the real top row (1000) + assertThat( + rows.stream() + .map(row -> row.getInt(0)) + .collect(java.util.stream.Collectors.toList())) + .contains(1000); + } + @Test void testColumnMaskingReadingSystemField() throws Exception { Identifier identifier = Identifier.create("test_table_db", "auth_table_masking_row_id"); @@ -4786,6 +4842,312 @@ void testColumnMaskingSystemFieldValidation() throws Exception { .hasMessageContaining("does not project"); } + @Test + void testColumnMaskingDisablesFilterStatsPruning() throws Exception { + Identifier identifier = + Identifier.create("test_table_db", "auth_table_masking_filter_stats"); + List fields = new ArrayList<>(); + fields.add(new DataField(0, "amount", DataTypes.INT())); + fields.add(new DataField(1, "src", DataTypes.INT())); + Table table = + createMaskingAuthTable( + identifier, + fields, + Collections.singletonMap( + CoreOptions.SOURCE_SPLIT_TARGET_SIZE.key(), "1 b")); + // two splits whose raw and masked values order oppositely + for (int[] row : new int[][] {{1, 1000}, {900, 10}}) { + BatchWriteBuilder writeBuilder = table.newBatchWriteBuilder(); + BatchTableWrite write = writeBuilder.newWrite(); + write.write(GenericRow.of(row[0], row[1])); + BatchTableCommit commit = writeBuilder.newCommit(); + commit.commit(write.prepareCommit()); + write.close(); + commit.close(); + } + Map masking = new HashMap<>(); + masking.put("amount", new FieldTransform(new FieldRef(1, "src", DataTypes.INT()))); + setColumnMasking(identifier, masking); + + // the predicate applies to masked values, with no engine help (no executeFilter): + // raw statistics must not prune the split whose masked value matches, and the + // raw-matching row must not come back + LeafPredicate amountFilter = + LeafPredicate.of( + new FieldTransform(new FieldRef(0, "amount", DataTypes.INT())), + GreaterThan.INSTANCE, + Collections.singletonList(500)); + ReadBuilder readBuilder = + table.newReadBuilder().withProjection(new int[] {0}).withFilter(amountFilter); + TableRead read = readBuilder.newRead(); + List rows = + collectRows( + read.createReader(readBuilder.newScan().plan().splits()), + table.rowType().project("amount")); + assertThat( + rows.stream() + .map(row -> row.getInt(0)) + .collect(java.util.stream.Collectors.toList())) + .containsExactly(1000); + } + + @Test + void testMaskGrowthOnPushedFilterColumn() throws Exception { + Identifier identifier = + Identifier.create("test_table_db", "auth_table_masking_pushed_filter"); + Table table = + createMaskingAuthTable( + identifier, stringFields("display", "other"), Collections.emptyMap()); + writeStringRow(table, "d1", "o1"); + + LeafPredicate displayFilter = + LeafPredicate.of( + new FieldTransform(new FieldRef(0, "display", DataTypes.STRING())), + Equal.INSTANCE, + Collections.singletonList(BinaryString.fromString("d1"))); + ReadBuilder readBuilder = + table.newReadBuilder().withProjection(new int[] {0}).withFilter(displayFilter); + TableScan scan = readBuilder.newScan(); + TableRead read = readBuilder.newRead(); + List plain = + collectRows( + read.createReader(scan.plan().splits()), + table.rowType().project("display")); + assertThat(plain).hasSize(1); + + // a mask now covers the filter column + Map masking = new HashMap<>(); + masking.put( + "display", + new ConcatTransform(Collections.singletonList(BinaryString.fromString("****")))); + setColumnMasking(identifier, masking); + + // this scan already pruned with the column's raw statistics: it must fail closed + assertThatThrownBy(scan::plan) + .isInstanceOf(IllegalStateException.class) + .hasMessageContaining("Recreate the scan"); + + // a fresh scan carries no raw pruning, and the existing reader evaluates the + // filter on the masked value: 'd1' no longer matches + List splits = readBuilder.newScan().plan().splits(); + List masked = + collectRows(read.createReader(splits), table.rowType().project("display")); + assertThat(masked).isEmpty(); + } + + @Test + void testDeferredFilterAppliesToPartitionListing() throws Exception { + Identifier identifier = Identifier.create("test_table_db", "filter_partition_listing"); + catalog.createDatabase(identifier.getDatabaseName(), true); + List fields = new ArrayList<>(); + fields.add(new DataField(0, "p", DataTypes.STRING())); + fields.add(new DataField(1, "v", DataTypes.STRING())); + catalog.createTable( + identifier, + new Schema( + fields, + Collections.singletonList("p"), + Collections.emptyList(), + Collections.emptyMap(), + ""), + true); + Table table = catalog.getTable(identifier); + writeStringRow(table, "a", "v1"); + writeStringRow(table, "b", "v2"); + + LeafPredicate partitionFilter = + LeafPredicate.of( + new FieldTransform(new FieldRef(0, "p", DataTypes.STRING())), + Equal.INSTANCE, + Collections.singletonList(BinaryString.fromString("a"))); + // partition listing bypasses plan(): the filter must still prune + assertThat( + table.newReadBuilder() + .withFilter(partitionFilter) + .newScan() + .listPartitionEntries()) + .hasSize(1); + } + + @Test + void testColumnMaskingDisablesLimitPushdown() throws Exception { + Identifier identifier = Identifier.create("test_table_db", "auth_table_masking_limit"); + List fields = new ArrayList<>(); + fields.add(new DataField(0, "amount", DataTypes.INT())); + fields.add(new DataField(1, "src", DataTypes.INT())); + Table table = + createMaskingAuthTable( + identifier, + fields, + Collections.singletonMap( + CoreOptions.SOURCE_SPLIT_TARGET_SIZE.key(), "1 b")); + for (int[] row : new int[][] {{900, 10}, {1, 1000}}) { + BatchWriteBuilder writeBuilder = table.newBatchWriteBuilder(); + BatchTableWrite write = writeBuilder.newWrite(); + write.write(GenericRow.of(row[0], row[1])); + BatchTableCommit commit = writeBuilder.newCommit(); + commit.commit(write.prepareCommit()); + write.close(); + commit.close(); + } + Map masking = new HashMap<>(); + masking.put("amount", new FieldTransform(new FieldRef(1, "src", DataTypes.INT()))); + setColumnMasking(identifier, masking); + + // the whole filter sits on the masked column: nothing is pushed down, but limit + // pruning must still know a read-time filter drops rows + LeafPredicate amountFilter = + LeafPredicate.of( + new FieldTransform(new FieldRef(0, "amount", DataTypes.INT())), + GreaterThan.INSTANCE, + Collections.singletonList(500)); + ReadBuilder readBuilder = + table.newReadBuilder() + .withProjection(new int[] {0}) + .withFilter(amountFilter) + .withLimit(1); + TableRead read = readBuilder.newRead().executeFilter(); + List rows = + collectRows( + read.createReader(readBuilder.newScan().plan().splits()), + table.rowType().project("amount")); + assertThat( + rows.stream() + .map(row -> row.getInt(0)) + .collect(java.util.stream.Collectors.toList())) + .contains(1000); + } + + @Test + void testFilterOnMaskedPartitionColumn() throws Exception { + Identifier identifier = Identifier.create("test_table_db", "auth_table_masking_partition"); + Table table = + createMaskingAuthTable( + identifier, + stringFields("p", "v"), + Collections.singletonList("p"), + Collections.emptyList(), + Collections.emptyMap()); + writeStringRow(table, "a", "va"); + writeStringRow(table, "b", "vb"); + + // the partition column is masked to another column's value + Map masking = new HashMap<>(); + masking.put("p", new FieldTransform(new FieldRef(1, "v", DataTypes.STRING()))); + setColumnMasking(identifier, masking); + + // engines consume partition filters without re-evaluating them, so the read + // itself must evaluate this predicate, on the masked value + LeafPredicate maskedMatch = + LeafPredicate.of( + new FieldTransform(new FieldRef(0, "p", DataTypes.STRING())), + Equal.INSTANCE, + Collections.singletonList(BinaryString.fromString("vb"))); + List rows = readWithFilter(table, maskedMatch, "p", "v"); + assertThat(rows).hasSize(1); + assertThat(rows.get(0).getString(0).toString()).isEqualTo("vb"); + assertThat(rows.get(0).getString(1).toString()).isEqualTo("vb"); + + // the raw partition value must not match: matching it would reveal the raw value + LeafPredicate rawMatch = + LeafPredicate.of( + new FieldTransform(new FieldRef(0, "p", DataTypes.STRING())), + Equal.INSTANCE, + Collections.singletonList(BinaryString.fromString("a"))); + assertThat(readWithFilter(table, rawMatch, "p", "v")).isEmpty(); + + // filter column not projected: it is read and masked for the filter only, then + // projected back out + List unprojected = readWithFilter(table, maskedMatch, "v"); + assertThat(unprojected).hasSize(1); + assertThat(unprojected.get(0).getString(0).toString()).isEqualTo("vb"); + } + + @Test + void testLimitWithFilterOnMaskedPartitionColumn() throws Exception { + Identifier identifier = + Identifier.create("test_table_db", "auth_table_masking_partition_limit"); + Table table = + createMaskingAuthTable( + identifier, + stringFields("p", "v"), + Collections.singletonList("p"), + Collections.emptyList(), + Collections.emptyMap()); + writeStringRow(table, "a", "va"); + writeStringRow(table, "b", "vb"); + Map masking = new HashMap<>(); + masking.put("p", new FieldTransform(new FieldRef(1, "v", DataTypes.STRING()))); + setColumnMasking(identifier, masking); + + // the filter is partition-only but sits on a masked column, so it drops rows at + // read time: limit pruning must not pick splits by raw row counts + LeafPredicate maskedMatch = + LeafPredicate.of( + new FieldTransform(new FieldRef(0, "p", DataTypes.STRING())), + Equal.INSTANCE, + Collections.singletonList(BinaryString.fromString("vb"))); + ReadBuilder readBuilder = + table.newReadBuilder() + .withProjection(new int[] {0, 1}) + .withFilter(maskedMatch) + .withLimit(1); + List rows = + collectRows( + readBuilder.newRead().createReader(readBuilder.newScan().plan().splits()), + table.rowType().project("p", "v")); + assertThat(rows).hasSize(1); + assertThat(rows.get(0).getString(1).toString()).isEqualTo("vb"); + } + + @Test + void testMaskedPkFilterNotAppliedOnRawValues() throws Exception { + Identifier identifier = Identifier.create("test_table_db", "auth_table_masking_pk_filter"); + List fields = new ArrayList<>(); + fields.add(new DataField(0, "id", DataTypes.INT().notNull())); + fields.add(new DataField(1, "src", DataTypes.INT())); + Table table = + createMaskingAuthTable( + identifier, + fields, + Collections.emptyList(), + Collections.singletonList("id"), + Collections.singletonMap(CoreOptions.BUCKET.key(), "1")); + BatchWriteBuilder writeBuilder = table.newBatchWriteBuilder(); + BatchTableWrite write = writeBuilder.newWrite(); + write.write(GenericRow.of(1, 500)); + write.write(GenericRow.of(2, 600)); + BatchTableCommit commit = writeBuilder.newCommit(); + commit.commit(write.prepareCommit()); + write.close(); + commit.close(); + + Map masking = new HashMap<>(); + masking.put("id", new FieldTransform(new FieldRef(1, "src", DataTypes.INT()))); + setColumnMasking(identifier, masking); + + // the key filter matches a masked value only: key-range skipping inside the + // merge read must not drop the row by its raw key + LeafPredicate idFilter = + LeafPredicate.of( + new FieldTransform(new FieldRef(0, "id", DataTypes.INT())), + Equal.INSTANCE, + Collections.singletonList(500)); + List rows = readWithFilter(table, idFilter, "id", "src"); + assertThat(rows).hasSize(1); + assertThat(rows.get(0).getInt(0)).isEqualTo(500); + } + + private List readWithFilter(Table table, Predicate filter, String... projected) + throws Exception { + int[] projection = table.rowType().getFieldIndices(Arrays.asList(projected)); + ReadBuilder readBuilder = + table.newReadBuilder().withProjection(projection).withFilter(filter); + return collectRows( + readBuilder.newRead().createReader(readBuilder.newScan().plan().splits()), + table.rowType().project(projected)); + } + private static void readFully(Table table) throws Exception { ReadBuilder readBuilder = table.newReadBuilder(); collectRows( @@ -5236,6 +5598,409 @@ void testRowFilterReadLimitSkippedWithTopN() throws Exception { assertThat(result).containsExactlyInAnyOrder("+I[3, 30]", "+I[4, 40]"); } + @Test + void testColumnMaskingOrFilterWithUnprojectedOperand() throws Exception { + Identifier identifier = Identifier.create("test_table_db", "auth_or_filter_unprojected"); + catalog.createDatabase(identifier.getDatabaseName(), true); + List fields = new ArrayList<>(); + fields.add(new DataField(0, "a", DataTypes.STRING())); + fields.add(new DataField(1, "b", DataTypes.STRING())); + fields.add(new DataField(2, "c", DataTypes.STRING())); + catalog.createTable( + identifier, + new Schema( + fields, + Collections.emptyList(), + Collections.emptyList(), + Collections.singletonMap(QUERY_AUTH_ENABLED.key(), "true"), + ""), + true); + Table table = catalog.getTable(identifier); + commitRows( + table, + GenericRow.of( + BinaryString.fromString("raw"), + BinaryString.fromString("bee"), + BinaryString.fromString("cee"))); + + // mask a -> "MASKED"; b and c are not masked + Map masking = new HashMap<>(); + masking.put( + "a", + new ConcatTransform(Collections.singletonList(BinaryString.fromString("MASKED")))); + setColumnMasking(identifier, masking); + + // WHERE a = 'MASKED' OR b = 'bee' -- one masked operand, one plain, neither projected. + // splitAnd does not split the OR, so retainFields keeps the whole disjunction, but only + // the masked column 'a' is widened in; 'b' is missing from the read schema. + Predicate onMasked = + LeafPredicate.of( + new FieldTransform(new FieldRef(0, "a", DataTypes.STRING())), + Equal.INSTANCE, + Collections.singletonList(BinaryString.fromString("MASKED"))); + Predicate onPlain = + LeafPredicate.of( + new FieldTransform(new FieldRef(1, "b", DataTypes.STRING())), + Equal.INSTANCE, + Collections.singletonList(BinaryString.fromString("bee"))); + Predicate disjunction = PredicateBuilder.or(onMasked, onPlain); + + ReadBuilder readBuilder = + table.newReadBuilder().withProjection(new int[] {2}).withFilter(disjunction); + List splits = readBuilder.newScan().plan().splits(); + List out = new ArrayList<>(); + try (RecordReader reader = readBuilder.newRead().createReader(splits)) { + reader.forEachRemaining(r -> out.add(r.getString(0).toString())); + } + assertThat(out).containsExactly("cee"); + } + + @Test + void testColumnMaskingPartitionListingBeforePlan() throws Exception { + Identifier identifier = Identifier.create("test_table_db", "auth_partition_listing"); + catalog.createDatabase(identifier.getDatabaseName(), true); + List fields = new ArrayList<>(); + fields.add(new DataField(0, "pt", DataTypes.STRING())); + fields.add(new DataField(1, "a", DataTypes.STRING())); + catalog.createTable( + identifier, + new Schema( + fields, + Collections.singletonList("pt"), + Collections.emptyList(), + Collections.singletonMap(QUERY_AUTH_ENABLED.key(), "true"), + ""), + true); + Table table = catalog.getTable(identifier); + commitRows( + table, + GenericRow.of(BinaryString.fromString("p1"), BinaryString.fromString("raw"))); + + Map masking = new HashMap<>(); + masking.put( + "a", + new ConcatTransform(Collections.singletonList(BinaryString.fromString("MASKED")))); + setColumnMasking(identifier, masking); + + Predicate onMasked = + LeafPredicate.of( + new FieldTransform(new FieldRef(1, "a", DataTypes.STRING())), + Equal.INSTANCE, + Collections.singletonList(BinaryString.fromString("MASKED"))); + + // listPartitionEntries() bypasses plan(); a later plan() on the same scan must not treat + // the masks discovered then as a rule change against an already-pushed filter. + InnerTableScan scan = + (InnerTableScan) table.newReadBuilder().withFilter(onMasked).newScan(); + assertThat(scan.listPartitionEntries()).hasSize(1); + assertThat(scan.plan().splits()).isNotEmpty(); + } + + @Test + void testQueryAuthLimitDoesNotCutRowsBeforeFilter() throws Exception { + Identifier identifier = Identifier.create("test_table_db", "auth_limit_with_filter"); + catalog.createDatabase(identifier.getDatabaseName(), true); + List fields = new ArrayList<>(); + fields.add(new DataField(0, "a", DataTypes.STRING())); + fields.add(new DataField(1, "b", DataTypes.STRING())); + catalog.createTable( + identifier, + new Schema( + fields, + Collections.emptyList(), + Collections.emptyList(), + Collections.singletonMap(QUERY_AUTH_ENABLED.key(), "true"), + ""), + true); + Table table = catalog.getTable(identifier); + // one file, the matching row second + commitRows( + table, + GenericRow.of(BinaryString.fromString("no"), BinaryString.fromString("r1")), + GenericRow.of(BinaryString.fromString("yes"), BinaryString.fromString("r2"))); + + // The reader does not evaluate the query filter on an auth-enabled table; engines + // re-apply it. A read-level limit would cap the raw rows at "no" and the engine would + // then filter it away, losing the matching row entirely. + Predicate onA = + LeafPredicate.of( + new FieldTransform(new FieldRef(0, "a", DataTypes.STRING())), + Equal.INSTANCE, + Collections.singletonList(BinaryString.fromString("yes"))); + ReadBuilder readBuilder = table.newReadBuilder().withFilter(onA).withLimit(1); + List out = new ArrayList<>(); + try (RecordReader reader = + readBuilder.newRead().createReader(readBuilder.newScan().plan().splits())) { + reader.forEachRemaining(r -> out.add(r.getString(1).toString())); + } + assertThat(out).contains("r2"); + } + + @Test + void testMaskedPartitionKeyRejectsPartitionFilter() throws Exception { + Identifier identifier = Identifier.create("test_table_db", "auth_masked_part_filter"); + Table table = + createMaskingAuthTable( + identifier, + stringFields("p", "v"), + Collections.singletonList("p"), + Collections.emptyList(), + Collections.emptyMap()); + writeStringRow(table, "a", "va"); + writeStringRow(table, "b", "vb"); + + Map masking = new HashMap<>(); + masking.put("p", new FieldTransform(new FieldRef(1, "v", DataTypes.STRING()))); + setColumnMasking(identifier, masking); + + LeafPredicate maskedMatch = + LeafPredicate.of( + new FieldTransform(new FieldRef(0, "p", DataTypes.STRING())), + Equal.INSTANCE, + Collections.singletonList(BinaryString.fromString("vb"))); + + // routed through withPartitionFilter (as Spark does) the predicate never reaches the + // masked value, so it must fail closed rather than prune on the raw partition value + InnerTableScan partitionScan = (InnerTableScan) table.newReadBuilder().newScan(); + partitionScan.withPartitionFilter(maskedMatch); + assertThatThrownBy(partitionScan::plan) + .isInstanceOf(UnsupportedOperationException.class) + .hasMessageContaining("masked partition key"); + + // the same predicate through withFilter is evaluated post-mask and still works + assertThat(readWithFilter(table, maskedMatch, "p", "v")).hasSize(1); + } + + @Test + void testPartitionFilterAllowedOnUnmaskedPartitionKey() throws Exception { + Identifier identifier = Identifier.create("test_table_db", "auth_two_part_keys"); + Table table = + createMaskingAuthTable( + identifier, + stringFields("p1", "p2", "v"), + Arrays.asList("p1", "p2"), + Collections.emptyList(), + Collections.emptyMap()); + writeStringRows(table, new String[] {"x", "a", "v1"}, new String[] {"y", "b", "v2"}); + + // only p2 is masked + Map masking = new HashMap<>(); + masking.put("p2", new FieldTransform(new FieldRef(2, "v", DataTypes.STRING()))); + setColumnMasking(identifier, masking); + + // a partition predicate touching only the UNMASKED key stays prunable + LeafPredicate onP1 = + LeafPredicate.of( + new FieldTransform(new FieldRef(0, "p1", DataTypes.STRING())), + Equal.INSTANCE, + Collections.singletonList(BinaryString.fromString("x"))); + InnerTableScan okScan = (InnerTableScan) table.newReadBuilder().newScan(); + okScan.withPartitionFilter(onP1); + assertThat(okScan.plan().splits()).isNotEmpty(); + + // touching the masked key is still rejected + LeafPredicate onP2 = + LeafPredicate.of( + new FieldTransform(new FieldRef(1, "p2", DataTypes.STRING())), + Equal.INSTANCE, + Collections.singletonList(BinaryString.fromString("v1"))); + InnerTableScan badScan = (InnerTableScan) table.newReadBuilder().newScan(); + badScan.withPartitionFilter(onP2); + assertThatThrownBy(badScan::plan) + .isInstanceOf(UnsupportedOperationException.class) + .hasMessageContaining("masked partition key"); + } + + @Test + void testQueryAuthLimitAppliesWhenReaderExecutesFilter() throws Exception { + Identifier identifier = Identifier.create("test_table_db", "auth_limit_execute_filter"); + catalog.createDatabase(identifier.getDatabaseName(), true); + List fields = new ArrayList<>(); + fields.add(new DataField(0, "a", DataTypes.STRING())); + fields.add(new DataField(1, "b", DataTypes.STRING())); + catalog.createTable( + identifier, + new Schema( + fields, + Collections.emptyList(), + Collections.emptyList(), + Collections.singletonMap(QUERY_AUTH_ENABLED.key(), "true"), + ""), + true); + Table table = catalog.getTable(identifier); + commitRows( + table, + GenericRow.of(BinaryString.fromString("yes"), BinaryString.fromString("r1")), + GenericRow.of(BinaryString.fromString("yes"), BinaryString.fromString("r2")), + GenericRow.of(BinaryString.fromString("yes"), BinaryString.fromString("r3"))); + + // executeFilter() means the reader evaluates the predicate itself, so capping after it + // is safe and the caller's limit must still be honoured + Predicate onA = + LeafPredicate.of( + new FieldTransform(new FieldRef(0, "a", DataTypes.STRING())), + Equal.INSTANCE, + Collections.singletonList(BinaryString.fromString("yes"))); + ReadBuilder readBuilder = table.newReadBuilder().withFilter(onA).withLimit(2); + List out = new ArrayList<>(); + try (RecordReader reader = + readBuilder + .newRead() + .executeFilter() + .createReader(readBuilder.newScan().plan().splits())) { + reader.forEachRemaining(r -> out.add(r.getString(1).toString())); + } + assertThat(out).hasSize(2); + } + + @Test + void testPartitionFilterFieldsReplacedOnRepush() throws Exception { + Identifier identifier = Identifier.create("test_table_db", "auth_part_filter_repush"); + Table table = + createMaskingAuthTable( + identifier, + stringFields("p1", "p2", "v"), + Arrays.asList("p1", "p2"), + Collections.emptyList(), + Collections.emptyMap()); + writeStringRows(table, new String[] {"x", "a", "v1"}); + + Map masking = new HashMap<>(); + masking.put("p2", new FieldTransform(new FieldRef(2, "v", DataTypes.STRING()))); + setColumnMasking(identifier, masking); + + LeafPredicate onMaskedP2 = + LeafPredicate.of( + new FieldTransform(new FieldRef(1, "p2", DataTypes.STRING())), + Equal.INSTANCE, + Collections.singletonList(BinaryString.fromString("v1"))); + LeafPredicate onPlainP1 = + LeafPredicate.of( + new FieldTransform(new FieldRef(0, "p1", DataTypes.STRING())), + Equal.INSTANCE, + Collections.singletonList(BinaryString.fromString("x"))); + + // the second push overwrites the first in ManifestsReader, so the tracked fields must be + // replaced too: the effective predicate only touches the unmasked key + InnerTableScan scan = (InnerTableScan) table.newReadBuilder().newScan(); + scan.withPartitionFilter(onMaskedP2); + scan.withPartitionFilter(onPlainP1); + assertThat(scan.plan().splits()).isNotEmpty(); + } + + @Test + void testPhysicalMetadataSystemTablesRejectedUnderQueryAuth() throws Exception { + Identifier identifier = Identifier.create("test_table_db", "auth_systab_physical"); + Table table = + createMaskingAuthTable( + identifier, stringFields("secret", "other"), Collections.emptyMap()); + writeStringRow(table, "TOPSECRET", "o1"); + Map masking = new HashMap<>(); + masking.put( + "secret", + new ConcatTransform(Collections.singletonList(BinaryString.fromString("****")))); + setColumnMasking(identifier, masking); + + // these report per-column min/max of the raw files, which no mask can cover + for (String suffix : Arrays.asList("files", "file_key_ranges", "binlog")) { + Identifier sysId = + Identifier.create( + identifier.getDatabaseName(), + identifier.getObjectName() + "$" + suffix); + assertThatThrownBy(() -> catalog.getTable(sysId)) + .isInstanceOf(UnsupportedOperationException.class) + .hasMessageContaining("query-auth table"); + } + + // the row-producing ones read through the masking reader and stay available + for (String suffix : Arrays.asList("audit_log", "ro")) { + Identifier sysId = + Identifier.create( + identifier.getDatabaseName(), + identifier.getObjectName() + "$" + suffix); + assertThat(batchRead(catalog.getTable(sysId)).toString()) + .contains("****") + .doesNotContain("TOPSECRET"); + } + } + + @Test + void testMaskReadingAnotherMaskedColumnRejected() throws Exception { + Identifier identifier = Identifier.create("test_table_db", "auth_mask_compose"); + Table table = + createMaskingAuthTable( + identifier, stringFields("secret", "display"), Collections.emptyMap()); + writeStringRow(table, "TOPSECRET", "ignored"); + + // display := secret, while secret is masked. A transform reads the raw row, so this + // would publish secret's raw value through display. + Map compose = new HashMap<>(); + compose.put( + "secret", + new ConcatTransform(Collections.singletonList(BinaryString.fromString("****")))); + compose.put("display", new FieldTransform(new FieldRef(0, "secret", DataTypes.STRING()))); + setColumnMasking(identifier, compose); + assertThatThrownBy(() -> batchRead(table)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("which is masked too"); + + // a mask reading an unmasked column, and one reading its own column, both stay valid + Map plain = new HashMap<>(); + plain.put("display", new FieldTransform(new FieldRef(0, "secret", DataTypes.STRING()))); + setColumnMasking(identifier, plain); + assertThat(batchRead(table).toString()).contains("TOPSECRET"); + + Map selfRef = new HashMap<>(); + selfRef.put( + "secret", + new UpperTransform( + Collections.singletonList(new FieldRef(0, "secret", DataTypes.STRING())))); + setColumnMasking(identifier, selfRef); + assertThat(batchRead(table).toString()).contains("TOPSECRET"); + } + + @Test + void testQueryAuthRejectedWhereItCannotBeEnforced() throws Exception { + // a non-file-store table never reads through the auth reader, so accepting the option + // would leave the rules silently inert + for (String type : Arrays.asList("format-table", "object-table")) { + Map opts = new HashMap<>(); + opts.put(QUERY_AUTH_ENABLED.key(), "true"); + opts.put("type", type); + opts.put("file.format", "csv"); + Identifier id = + Identifier.create( + "test_table_db", "auth_unsupported_" + type.replace('-', '_')); + assertThatThrownBy( + () -> + catalog.createTable( + id, + new Schema( + stringFields("secret"), + Collections.emptyList(), + Collections.emptyList(), + opts, + ""), + false)) + .hasMessageContaining(QUERY_AUTH_ENABLED.key()); + } + + // search ranks raw index values, so it is refused rather than answered from them + Identifier identifier = Identifier.create("test_table_db", "auth_search_rejected"); + Table table = + createMaskingAuthTable( + identifier, stringFields("secret", "other"), Collections.emptyMap()); + assertThatThrownBy( + () -> ((FileStoreTable) table).newFullTextSearchBuilder().newFullTextScan()) + .isInstanceOf(UnsupportedOperationException.class) + .hasMessageContaining("query-auth table"); + + // the lookup cache serves rows straight from the store + assertThatThrownBy(() -> new LocalTableQuery((FileStoreTable) table)) + .isInstanceOf(UnsupportedOperationException.class) + .hasMessageContaining("query-auth table"); + } + @Test void testRowFilterWithTopNKeepsAuthorizedSplits() throws Exception { Identifier identifier = Identifier.create("test_table_db", "auth_table_topn"); diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/RESTCatalogITCase.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/RESTCatalogITCase.java index 7e1f33136dd2..d1276136ee84 100644 --- a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/RESTCatalogITCase.java +++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/RESTCatalogITCase.java @@ -410,6 +410,48 @@ public void testColumnMaskingCrossColumnWithProjection() { .containsExactlyInAnyOrder(Row.of("o1"), Row.of("o2")); } + @Test + public void testFilterOnMaskedPartitionColumn() { + String maskingTable = "partition_masking_table"; + batchSql( + String.format( + "CREATE TABLE %s.%s (p STRING, v STRING) PARTITIONED BY (p)" + + " WITH ('query-auth.enabled' = 'true')", + DATABASE_NAME, maskingTable)); + batchSql( + String.format( + "INSERT INTO %s.%s VALUES ('a', 'va'), ('b', 'vb')", + DATABASE_NAME, maskingTable)); + + Map columnMasking = new HashMap<>(); + columnMasking.put("p", new FieldTransform(new FieldRef(1, "v", DataTypes.STRING()))); + restCatalogServer.setColumnMaskingAuth( + Identifier.create(DATABASE_NAME, maskingTable), columnMasking); + + // Flink consumes bounded partition filters without re-evaluating them, so the + // source itself must evaluate the predicate, on the masked value + assertThat( + batchSql( + String.format( + "SELECT p, v FROM %s.%s WHERE p = 'vb'", + DATABASE_NAME, maskingTable))) + .containsExactlyInAnyOrder(Row.of("vb", "vb")); + // the raw partition value must not match + assertThat( + batchSql( + String.format( + "SELECT p, v FROM %s.%s WHERE p = 'a'", + DATABASE_NAME, maskingTable))) + .isEmpty(); + // filter column not projected + assertThat( + batchSql( + String.format( + "SELECT v FROM %s.%s WHERE p = 'vb'", + DATABASE_NAME, maskingTable))) + .containsExactlyInAnyOrder(Row.of("vb")); + } + @Test public void testRowFilter() { String filterTable = "row_filter_table"; diff --git a/paimon-spark/paimon-spark-ut/src/test/java/org/apache/paimon/spark/SparkCatalogWithRestTest.java b/paimon-spark/paimon-spark-ut/src/test/java/org/apache/paimon/spark/SparkCatalogWithRestTest.java index 51c574857816..8eaf387b9a49 100644 --- a/paimon-spark/paimon-spark-ut/src/test/java/org/apache/paimon/spark/SparkCatalogWithRestTest.java +++ b/paimon-spark/paimon-spark-ut/src/test/java/org/apache/paimon/spark/SparkCatalogWithRestTest.java @@ -399,6 +399,27 @@ public void testColumnMaskingCrossColumnWithProjection() { .isEqualTo("[[o1], [o2]]"); } + @Test + public void testRowFilterDisablesAggregatePushdown() { + spark.sql( + "CREATE TABLE t_agg_pushdown (id INT) TBLPROPERTIES" + + " ('query-auth.enabled'='true')"); + spark.sql("INSERT INTO t_agg_pushdown VALUES (1), (2), (3)"); + + LeafPredicate idFilter = + LeafPredicate.of( + new FieldTransform(new FieldRef(0, "id", DataTypes.INT())), + GreaterThan.INSTANCE, + Collections.singletonList(1)); + restCatalogServer.setRowFilterAuth( + Identifier.create("db2", "t_agg_pushdown"), Collections.singletonList(idFilter)); + + // statistics-based aggregate pushdown must not bypass the read-time row filter + // (today it degrades because auth splits are not DataSplits; this anchors that) + assertThat(spark.sql("SELECT COUNT(*) FROM t_agg_pushdown").collectAsList().toString()) + .isEqualTo("[[2]]"); + } + @Test public void testRowFilter() { spark.sql( From 72b3fe709ffbfa034621df857d4335fb806cf658 Mon Sep 17 00:00:00 2001 From: Li Jiajia Date: Thu, 30 Jul 2026 06:02:54 -0400 Subject: [PATCH 3/7] [core] Fix two crashes the query-auth read path introduced Deferring the filter to the wrapped scan on a data-evolution table dropped the row-id-safe residual that `DataEvolutionBatchScan` used to pass alongside it. The wrapped scan strips masked columns but not row ids, so a `_ROW_ID` predicate reached statistics that carry logical columns only and planning threw ArrayIndexOutOfBounds -- with no masking rule configured at all. Strip the row-id part before deferring. The post-mask filter was remapped positionally against the table schema, which does not contain system fields. A masked `_ROW_ID` used in the predicate resolved to -1 and the read failed even though the field was in the emitted schema. Remap by name against that schema instead, as the rest of the auth path does. --- .../globalindex/DataEvolutionBatchScan.java | 9 +++-- .../table/source/AbstractDataTableRead.java | 13 +++---- .../apache/paimon/rest/RESTCatalogTest.java | 34 +++++++++++++++++++ 3 files changed, 47 insertions(+), 9 deletions(-) diff --git a/paimon-core/src/main/java/org/apache/paimon/globalindex/DataEvolutionBatchScan.java b/paimon-core/src/main/java/org/apache/paimon/globalindex/DataEvolutionBatchScan.java index b488c73664b0..b76abfc6f88d 100644 --- a/paimon-core/src/main/java/org/apache/paimon/globalindex/DataEvolutionBatchScan.java +++ b/paimon-core/src/main/java/org/apache/paimon/globalindex/DataEvolutionBatchScan.java @@ -97,9 +97,12 @@ public InnerTableScan withFilter(Predicate predicate) { this.filter = predicate; if (queryAuthEnabled()) { - // let the wrapped scan defer the filter: a conjunct on a masked column must not - // reach raw statistics or the global index - batchScan.withFilter(predicate); + // the wrapped scan defers the filter but strips only masked columns; row ids must + // go here, since data-evolution statistics carry logical columns only + Predicate residual = rowIdSafeResidualFilter(predicate); + if (residual != null) { + batchScan.withFilter(residual); + } return this; } batchScan.snapshotReader().withFilter(predicate, rowIdSafeResidualFilter(predicate)); diff --git a/paimon-core/src/main/java/org/apache/paimon/table/source/AbstractDataTableRead.java b/paimon-core/src/main/java/org/apache/paimon/table/source/AbstractDataTableRead.java index 64c5ea4a6be3..70e8ff0abfa6 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/source/AbstractDataTableRead.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/source/AbstractDataTableRead.java @@ -261,17 +261,18 @@ private RecordReader filterMaskedConjuncts( if (maskedPart == null) { return reader; } - int[] projection = schema.logicalRowType().getFieldIndices(outputType.getFieldNames()); - Optional remapped = - maskedPart.visit(PredicateProjectionConverter.fromProjection(projection)); - if (!remapped.isPresent()) { + // by name against the emitted schema: it may carry system fields the table schema lacks + Predicate filter; + try { + filter = TableQueryAuthResult.remapPredicate(maskedPart, outputType); + } catch (RuntimeException e) { throw new IllegalStateException( "Filter on masked columns " + maskedFilterFields + " cannot be evaluated on read schema " - + outputType.getFieldNames()); + + outputType.getFieldNames(), + e); } - Predicate filter = remapped.get(); return reader.filter(filter::test); } diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTest.java b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTest.java index 618185e3637b..db966fbce5b2 100644 --- a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTest.java @@ -80,6 +80,7 @@ import org.apache.paimon.table.FileStoreTable; import org.apache.paimon.table.FormatTable; import org.apache.paimon.table.Instant; +import org.apache.paimon.table.SpecialFields; import org.apache.paimon.table.Table; import org.apache.paimon.table.TableSnapshot; import org.apache.paimon.table.object.ObjectTable; @@ -6001,6 +6002,39 @@ void testQueryAuthRejectedWhereItCannotBeEnforced() throws Exception { .hasMessageContaining("query-auth table"); } + @Test + void testRowIdFilterOnDataEvolutionQueryAuthTable() throws Exception { + Identifier identifier = Identifier.create("test_table_db", "auth_de_rowid_filter"); + List fields = new ArrayList<>(); + fields.add(new DataField(0, "f0", DataTypes.INT())); + fields.add(new DataField(1, "f1", DataTypes.STRING())); + Map options = new HashMap<>(); + options.put(CoreOptions.ROW_TRACKING_ENABLED.key(), "true"); + options.put(CoreOptions.DATA_EVOLUTION_ENABLED.key(), "true"); + Table table = createMaskingAuthTable(identifier, fields, options); + + BatchWriteBuilder builder = table.newBatchWriteBuilder(); + try (BatchTableWrite write = builder.newWrite()) { + write.write(GenericRow.of(0, BinaryString.fromString("a0"))); + write.write(GenericRow.of(1, BinaryString.fromString("a1"))); + builder.newCommit().commit(write.prepareCommit()); + } + + // no masking rules at all -- a _ROW_ID predicate must still plan. Data-evolution + // statistics carry only logical columns, so the row-id part must not be pushed. + Predicate onRowId = + LeafPredicate.of( + new FieldTransform( + new FieldRef( + SpecialFields.ROW_ID.id(), + SpecialFields.ROW_ID.name(), + DataTypes.BIGINT())), + Equal.INSTANCE, + Collections.singletonList(0L)); + ReadBuilder readBuilder = table.newReadBuilder().withFilter(onRowId); + assertThat(readBuilder.newScan().plan().splits()).isNotEmpty(); + } + @Test void testRowFilterWithTopNKeepsAuthorizedSplits() throws Exception { Identifier identifier = Identifier.create("test_table_db", "auth_table_topn"); From e6b545413d4849e8133aae0c11426103b59e5123 Mon Sep 17 00:00:00 2001 From: Li Jiajia Date: Fri, 31 Jul 2026 11:21:51 -0400 Subject: [PATCH 4/7] [core] Keep the caller's partition filter under query auth Query auth defers the filter push to plan(), where the filter's partition conjuncts land in the same ManifestsReader slot the caller's own partition filter uses -- and overwrite it, since that slot is assigned rather than anded. ReadBuilderImpl pushes the partition filter after withFilter precisely to make it win, so the deferral silently reverses which one applies: a read combining both returned the partitions the caller had excluded. Re-apply it right after the deferred push. Off the auth path nothing is deferred, so the order stays as it was. --- .../table/source/AbstractDataTableScan.java | 44 +++++++++------- .../apache/paimon/rest/RESTCatalogTest.java | 51 +++++++++++++++++++ 2 files changed, 76 insertions(+), 19 deletions(-) diff --git a/paimon-core/src/main/java/org/apache/paimon/table/source/AbstractDataTableScan.java b/paimon-core/src/main/java/org/apache/paimon/table/source/AbstractDataTableScan.java index a86f19b014c2..eee6188a2266 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/source/AbstractDataTableScan.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/source/AbstractDataTableScan.java @@ -109,6 +109,8 @@ abstract class AbstractDataTableScan implements DataTableScan { private boolean filterPushed = false; private Set pushedMaskedFields = Collections.emptySet(); private Set partitionFilterFields = Collections.emptySet(); + // re-applied after the deferred filter push, which writes the same slot + @Nullable private Runnable repushPartitionFilter; protected AbstractDataTableScan( TableSchema schema, @@ -209,21 +211,19 @@ public InnerTableScan withReadType(@Nullable RowType readType) { @Override public AbstractDataTableScan withPartitionFilter(Map partitionSpec) { - partitionFilterFields = + return pushPartitionFilter( partitionSpec == null ? Collections.emptySet() - : new HashSet<>(partitionSpec.keySet()); - snapshotReader.withPartitionFilter(partitionSpec); - return this; + : new HashSet<>(partitionSpec.keySet()), + () -> snapshotReader.withPartitionFilter(partitionSpec)); } @Override public AbstractDataTableScan withPartitionFilter(List partitions) { // binary partitions carry no field names; assume every partition key - partitionFilterFields = - partitions == null ? Collections.emptySet() : new HashSet<>(schema.partitionKeys()); - snapshotReader.withPartitionFilter(partitions); - return this; + return pushPartitionFilter( + partitions == null ? Collections.emptySet() : new HashSet<>(schema.partitionKeys()), + () -> snapshotReader.withPartitionFilter(partitions)); } @Override @@ -232,29 +232,25 @@ public AbstractDataTableScan withPartitionsFilter(List> part if (partitions != null) { partitions.forEach(spec -> fields.addAll(spec.keySet())); } - partitionFilterFields = fields; - snapshotReader.withPartitionsFilter(partitions); - return this; + return pushPartitionFilter(fields, () -> snapshotReader.withPartitionsFilter(partitions)); } @Override public AbstractDataTableScan withPartitionFilter(PartitionPredicate partitionPredicate) { - partitionFilterFields = + return pushPartitionFilter( partitionPredicate == null ? Collections.emptySet() - : partitionPredicateFields(partitionPredicate); - snapshotReader.withPartitionFilter(partitionPredicate); - return this; + : partitionPredicateFields(partitionPredicate), + () -> snapshotReader.withPartitionFilter(partitionPredicate)); } @Override public InnerTableScan withPartitionFilter(Predicate predicate) { - partitionFilterFields = + return pushPartitionFilter( predicate == null ? Collections.emptySet() - : PredicateVisitor.collectFieldNames(predicate); - snapshotReader.withPartitionFilter(predicate); - return this; + : PredicateVisitor.collectFieldNames(predicate), + () -> snapshotReader.withPartitionFilter(predicate)); } @Override @@ -306,6 +302,13 @@ public InnerTableScan withRowRangeIndex(RowRangeIndex rowRangeIndex) { return this; } + private AbstractDataTableScan pushPartitionFilter(Set fields, Runnable push) { + partitionFilterFields = fields; + repushPartitionFilter = push; + push.run(); + return this; + } + /** The partition columns a predicate references, or all of them when it cannot be read. */ private Set partitionPredicateFields(PartitionPredicate partitionPredicate) { if (partitionPredicate instanceof PartitionPredicate.DefaultPartitionPredicate) { @@ -360,6 +363,9 @@ protected final void ensureFilterPushdown(Set maskedFields) { snapshotReader.withFilter(userFilter, effective); filterPushed = true; pushedMaskedFields = maskedInFilter; + if (repushPartitionFilter != null) { + repushPartitionFilter.run(); + } } else if (!pushedMaskedFields.containsAll(maskedInFilter)) { throw new IllegalStateException( "Query auth rules changed and now mask a pushed-down filter column. " diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTest.java b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTest.java index db966fbce5b2..0b628cb1a8b8 100644 --- a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTest.java @@ -5812,6 +5812,57 @@ void testPartitionFilterAllowedOnUnmaskedPartitionKey() throws Exception { .hasMessageContaining("masked partition key"); } + @Test + void testExplicitPartitionFilterSurvivesDeferredQueryFilter() throws Exception { + Identifier identifier = Identifier.create("test_table_db", "auth_part_filter_with_filter"); + Table table = + createMaskingAuthTable( + identifier, + stringFields("p", "v"), + Collections.singletonList("p"), + Collections.emptyList(), + Collections.emptyMap()); + writeStringRows(table, new String[] {"a", "va"}, new String[] {"b", "vb"}); + + // no masking: query auth alone defers the filter push, which must not overwrite + // the caller's partition filter + LeafPredicate pAtLeastA = + LeafPredicate.of( + new FieldTransform(new FieldRef(0, "p", DataTypes.STRING())), + GreaterOrEqual.INSTANCE, + Collections.singletonList(BinaryString.fromString("a"))); + assertThat(readPartitionB(table, pAtLeastA)).containsExactly("b"); + + // control: without query auth the filter is pushed eagerly, so this always held + Identifier plain = Identifier.create("test_table_db", "plain_part_filter_with_filter"); + catalog.createTable( + plain, + new Schema( + stringFields("p", "v"), + Collections.singletonList("p"), + Collections.emptyList(), + Collections.emptyMap(), + ""), + true); + Table plainTable = catalog.getTable(plain); + writeStringRows(plainTable, new String[] {"a", "va"}, new String[] {"b", "vb"}); + assertThat(readPartitionB(plainTable, pAtLeastA)).containsExactly("b"); + } + + private static List readPartitionB(Table table, Predicate filter) throws Exception { + ReadBuilder readBuilder = + table.newReadBuilder() + .withFilter(filter) + .withPartitionFilter(Collections.singletonMap("p", "b")); + return collectRows( + readBuilder.newRead().createReader(readBuilder.newScan().plan().splits()), + table.rowType()) + .stream() + .map(row -> row.getString(0).toString()) + .distinct() + .collect(Collectors.toList()); + } + @Test void testQueryAuthLimitAppliesWhenReaderExecutesFilter() throws Exception { Identifier identifier = Identifier.create("test_table_db", "auth_limit_execute_filter"); From f4f9c0313d740bf287efd4004373a9640405dc97 Mon Sep 17 00:00:00 2001 From: Li Jiajia Date: Fri, 31 Jul 2026 11:55:30 -0400 Subject: [PATCH 5/7] [core] Tidy up the query-auth read path The scan and the read each computed the same set of columns to widen the projection by, in the same order, from the same inputs -- and they have to agree: the read schema is fixed on first use, so a scan that widens less than the read wants makes the read throw. Keep it in one place, and narrow what that leaves unused to private. Drops widenReadType, which nothing outside its own tests called. Also drops the step-by-step commentary from the tests, whose names already say what each case covers. --- .../paimon/catalog/TableQueryAuthResult.java | 36 ++++-- .../table/source/AbstractDataTableRead.java | 17 +-- .../table/source/AbstractDataTableScan.java | 28 ++--- .../catalog/TableQueryAuthResultTest.java | 17 --- .../operation/MergeFileSplitReadTest.java | 7 -- .../apache/paimon/rest/RESTCatalogTest.java | 110 +----------------- .../paimon/flink/RESTCatalogITCase.java | 8 -- .../spark/SparkCatalogWithRestTest.java | 6 - 8 files changed, 34 insertions(+), 195 deletions(-) diff --git a/paimon-core/src/main/java/org/apache/paimon/catalog/TableQueryAuthResult.java b/paimon-core/src/main/java/org/apache/paimon/catalog/TableQueryAuthResult.java index d29853509787..166a702db234 100644 --- a/paimon-core/src/main/java/org/apache/paimon/catalog/TableQueryAuthResult.java +++ b/paimon-core/src/main/java/org/apache/paimon/catalog/TableQueryAuthResult.java @@ -93,16 +93,6 @@ public boolean hasRules() { return extractPredicate() != null || !extractColumnMasking().isEmpty(); } - /** - * Widens {@code readType} with the unprojected columns the rules read, or null when the - * projection already covers them. Scans apply this before planning file pruning. - */ - @Nullable - public RowType widenReadType(RowType tableType, RowType readType) { - return appendMissingFields( - tableType, readType, requiredAuthFields(readType.getFieldNames())); - } - /** * Drops the conjuncts of {@code predicate} referencing any of {@code fields}; returns null when * nothing remains. Used to keep raw-statistics pushdown off masked columns. @@ -141,7 +131,7 @@ private static Predicate filterConjuncts( * Every column read by the conjuncts of {@code filter} that touch {@code maskTargets}. Their * unmasked operands count too, since splitAnd does not split a disjunction. */ - public static Set postMaskFilterFields( + private static Set postMaskFilterFields( @Nullable Predicate filter, Set maskTargets) { if (filter == null || maskTargets.isEmpty()) { return Collections.emptySet(); @@ -157,6 +147,28 @@ public static Set postMaskFilterFields( : new HashSet<>(PredicateVisitor.collectFieldNames(retained)); } + /** + * The columns a read projecting {@code readFields} under {@code filter} must additionally + * expose: the rule fields, plus the operands of the conjuncts evaluated post-mask. The scan and + * the read both widen by this, and must agree — the read schema is fixed on first use. + */ + public Set authFields(List readFields, @Nullable Predicate filter) { + Set postMask = postMaskFilterFields(filter, extractColumnMasking().keySet()); + List visible = readFields; + if (!postMask.isEmpty()) { + visible = new ArrayList<>(readFields); + for (String field : postMask) { + if (!visible.contains(field)) { + visible.add(field); + } + } + } + Set ruleFields = requiredAuthFields(visible); + // requiredAuthFields returns what the rules read, not the operands themselves + ruleFields.addAll(postMask); + return ruleFields; + } + /** Appends the missing {@code ruleFields} of {@code tableType} to {@code readType}. */ @Nullable public static RowType appendMissingFields( @@ -340,7 +352,7 @@ private static void checkNotRenamed( * The field names the auth rules read for a query projecting {@code projectedFields}: the * row-filter operands, plus transitively the inputs of every mask whose target is readable. */ - public Set requiredAuthFields(List projectedFields) { + private Set requiredAuthFields(List projectedFields) { Map masking = extractColumnMasking(); Set ruleFields = new HashSet<>(); Set readable = new HashSet<>(projectedFields); diff --git a/paimon-core/src/main/java/org/apache/paimon/table/source/AbstractDataTableRead.java b/paimon-core/src/main/java/org/apache/paimon/table/source/AbstractDataTableRead.java index 70e8ff0abfa6..38474d0fa517 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/source/AbstractDataTableRead.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/source/AbstractDataTableRead.java @@ -35,7 +35,6 @@ import javax.annotation.Nullable; import java.io.IOException; -import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; @@ -178,21 +177,7 @@ private RecordReader authedReader(Split split, TableQueryAuthResult // masked filter columns are read and masked like rule fields, then evaluated post-mask Set maskedFilterFields = maskedFilterFields(authResult.extractColumnMasking().keySet()); - // a retained conjunct may also reference unmasked columns; all must be readable - Set postMaskFilterFields = - TableQueryAuthResult.postMaskFilterFields( - predicate, authResult.extractColumnMasking().keySet()); - List visibleFields = readFields; - if (!postMaskFilterFields.isEmpty()) { - visibleFields = new ArrayList<>(readFields); - for (String field : postMaskFilterFields) { - if (!visibleFields.contains(field)) { - visibleFields.add(field); - } - } - } - Set ruleFields = authResult.requiredAuthFields(visibleFields); - ruleFields.addAll(postMaskFilterFields); + Set ruleFields = authResult.authFields(readFields, predicate); RowType widened = widenedReadType(authResult, ruleFields); if (widened != null && !widened.equals(appliedReadType)) { applyReadType(widened); diff --git a/paimon-core/src/main/java/org/apache/paimon/table/source/AbstractDataTableScan.java b/paimon-core/src/main/java/org/apache/paimon/table/source/AbstractDataTableScan.java index eee6188a2266..c35e27828230 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/source/AbstractDataTableScan.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/source/AbstractDataTableScan.java @@ -68,7 +68,6 @@ import javax.annotation.Nullable; -import java.util.ArrayList; import java.util.Collections; import java.util.HashSet; import java.util.List; @@ -110,7 +109,7 @@ abstract class AbstractDataTableScan implements DataTableScan { private Set pushedMaskedFields = Collections.emptySet(); private Set partitionFilterFields = Collections.emptySet(); // re-applied after the deferred filter push, which writes the same slot - @Nullable private Runnable repushPartitionFilter; + @Nullable private Runnable reapplyPartitionFilter; protected AbstractDataTableScan( TableSchema schema, @@ -304,7 +303,7 @@ public InnerTableScan withRowRangeIndex(RowRangeIndex rowRangeIndex) { private AbstractDataTableScan pushPartitionFilter(Set fields, Runnable push) { partitionFilterFields = fields; - repushPartitionFilter = push; + reapplyPartitionFilter = push; push.run(); return this; } @@ -363,8 +362,8 @@ protected final void ensureFilterPushdown(Set maskedFields) { snapshotReader.withFilter(userFilter, effective); filterPushed = true; pushedMaskedFields = maskedInFilter; - if (repushPartitionFilter != null) { - repushPartitionFilter.run(); + if (reapplyPartitionFilter != null) { + reapplyPartitionFilter.run(); } } else if (!pushedMaskedFields.containsAll(maskedInFilter)) { throw new IllegalStateException( @@ -384,24 +383,11 @@ private void applyAuthReadType(@Nullable TableQueryAuthResult queryAuthResult) { RowType desired = readType; if (queryAuthResult != null && queryAuthResult.hasRules()) { // post-mask conjuncts are evaluated at read time; their columns must survive planning - List seed = readType.getFieldNames(); - Set postMask = - TableQueryAuthResult.postMaskFilterFields( - userFilter, queryAuthResult.extractColumnMasking().keySet()); - if (!postMask.isEmpty()) { - seed = new ArrayList<>(seed); - for (String field : postMask) { - if (!seed.contains(field)) { - seed.add(field); - } - } - } - Set ruleFields = queryAuthResult.requiredAuthFields(seed); - // requiredAuthFields returns what the rules read, not the seed itself - ruleFields.addAll(postMask); RowType widened = TableQueryAuthResult.appendMissingFields( - schema.logicalRowType(), readType, ruleFields); + schema.logicalRowType(), + readType, + queryAuthResult.authFields(readType.getFieldNames(), userFilter)); if (widened != null) { desired = widened; } diff --git a/paimon-core/src/test/java/org/apache/paimon/catalog/TableQueryAuthResultTest.java b/paimon-core/src/test/java/org/apache/paimon/catalog/TableQueryAuthResultTest.java index 4573ceae83e8..5b7f64fe8a59 100644 --- a/paimon-core/src/test/java/org/apache/paimon/catalog/TableQueryAuthResultTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/catalog/TableQueryAuthResultTest.java @@ -122,13 +122,9 @@ public void testValidateRejectsReAddedColumnOfSameName() { Map masking = Collections.singletonMap("display", maskJson()); TableQueryAuthResult result = new TableQueryAuthResult(null, masking); - // same names and ids: the rule binds to the same physical columns assertThatCode(() -> result.validateReadableWithoutRename(TABLE_TYPE, TABLE_TYPE)) .doesNotThrowAnyException(); - // the mask input 'extra' was dropped and re-added, so the latest schema gives it a fresh - // id. A time-travel read of the pre-drop snapshot still has an 'extra', but it is an - // unrelated column; resolving the rule by name would mask with its values. RowType latest = RowType.of( new org.apache.paimon.types.DataField(0, "display", DataTypes.STRING()), @@ -140,7 +136,6 @@ public void testValidateRejectsReAddedColumnOfSameName() { @Test public void testValidateRejectsReAddedColumnForRowFilter() { - // a row filter is remapped by name too, so it needs the same binding check as a mask TableQueryAuthResult result = new TableQueryAuthResult(Collections.singletonList(filterJson()), null); assertThatCode(() -> result.validateReadableWithoutRename(TABLE_TYPE, TABLE_TYPE)) @@ -167,16 +162,4 @@ public void testHasRules() { Map masking = Collections.singletonMap("display", maskJson()); assertThat(new TableQueryAuthResult(null, masking).hasRules()).isTrue(); } - - @Test - public void testWidenReadType() { - Map masking = Collections.singletonMap("display", maskJson()); - TableQueryAuthResult result = new TableQueryAuthResult(null, masking); - // the mask input is unprojected: widen - RowType widened = result.widenReadType(TABLE_TYPE, TABLE_TYPE.project("display")); - assertThat(widened).isNotNull(); - assertThat(widened.getFieldNames()).containsExactly("display", "extra"); - // already covered: no widening - assertThat(result.widenReadType(TABLE_TYPE, TABLE_TYPE)).isNull(); - } } diff --git a/paimon-core/src/test/java/org/apache/paimon/operation/MergeFileSplitReadTest.java b/paimon-core/src/test/java/org/apache/paimon/operation/MergeFileSplitReadTest.java index ce39cf6303c2..cdabdaf1d80c 100644 --- a/paimon-core/src/test/java/org/apache/paimon/operation/MergeFileSplitReadTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/operation/MergeFileSplitReadTest.java @@ -328,7 +328,6 @@ record -> record.value().getString(1).toString())); @Test public void testRepeatedReadTypeResetsOuterProjection() throws Exception { - // a second withReadType that needs no adjustment must clear the outer projection TestKeyValueGenerator gen = new TestKeyValueGenerator(); List data = new ArrayList<>(); for (int i = 0; i < 100; i++) { @@ -351,9 +350,7 @@ public void testRepeatedReadTypeResetsOuterProjection() throws Exception { .collect(Collectors.groupingBy(ManifestEntry::partition)); MergeFileSplitRead read = store.newRead(); - // adjusted internally to include the sequence field: outer projection set read.withReadType(TestKeyValueGenerator.DEFAULT_ROW_TYPE.project("shopId", "dt", "hr")); - // contains the sequence field, no adjustment: previous outer projection cleared read.withReadType( TestKeyValueGenerator.DEFAULT_ROW_TYPE.project("shopId", "dt", "hr", "orderId")); @@ -448,8 +445,6 @@ private static KeyValue keyValue( @Test public void testIncrementalDiffReadOnProjectedMergeRead() throws Exception { - // the diff read projects the merge read's output; when the shared merge read - // is itself projected, the projection base must be its actual output type TestKeyValueGenerator gen = new TestKeyValueGenerator(); List before = new ArrayList<>(); for (int i = 0; i < 50; i++) { @@ -476,7 +471,6 @@ public void testIncrementalDiffReadOnProjectedMergeRead() throws Exception { .collect(Collectors.groupingBy(ManifestEntry::partition)); MergeFileSplitRead mergeRead = store.newRead(); - // out-of-table-order projection, pushed into the shared merge read RowType projection = TestKeyValueGenerator.DEFAULT_ROW_TYPE.project("shopId", "dt", "hr"); mergeRead.withReadType(projection); SplitRead diffRead = new IncrementalDiffSplitRead(mergeRead); @@ -501,7 +495,6 @@ public void testIncrementalDiffReadOnProjectedMergeRead() throws Exception { while (iterator.hasNext()) { InternalRow row = iterator.next(); assertThat(row.getFieldCount()).isEqualTo(3); - // shopId INT, dt STRING(len 8), hr INT: misprojection would misplace types assertThat(row.getString(1).toString()).hasSize(8); row.getInt(0); row.getInt(2); diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTest.java b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTest.java index 0b628cb1a8b8..4c3f77f77cf4 100644 --- a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTest.java @@ -3956,7 +3956,6 @@ private static void writeStringRow(Table table, String... values) throws Excepti writeStringRows(table, values); } - /** Cross-column mask: display := concat_ws('-', first, last). */ private void maskDisplayWithFullName(Identifier identifier) { Map columnMasking = new HashMap<>(); columnMasking.put( @@ -3969,7 +3968,6 @@ private void maskDisplayWithFullName(Identifier identifier) { setColumnMasking(identifier, columnMasking); } - /** Rows must be copied: the auth back-projection reuses one ProjectedRow per split. */ private static List collectRows(RecordReader reader, RowType rowType) throws Exception { List rows = new ArrayList<>(); @@ -3986,14 +3984,12 @@ void testColumnMaskingCrossColumnWithProjection() throws Exception { identifier, stringFields("first", "last", "display", "other"), Collections.emptyMap()); - // two rows in one commit -> one split with multiple rows writeStringRows( table, new String[] {"john", "doe", "ignored", "o1"}, new String[] {"jane", "roe", "ignored", "o2"}); maskDisplayWithFullName(identifier); - // project only the masked target; its input columns are not selected ReadBuilder readBuilder = table.newReadBuilder().withProjection(new int[] {2}); List splits = readBuilder.newScan().plan().splits(); List rows = @@ -4020,10 +4016,8 @@ void testColumnMaskingProjectionAcrossMultipleSplits() throws Exception { createMaskingAuthTable( identifier, stringFields("first", "last", "display"), - // one file per split, so the scan below yields multiple splits Collections.singletonMap( CoreOptions.SOURCE_SPLIT_TARGET_SIZE.key(), "1 b")); - // two commits -> two splits; the widening must not leak state across splits writeStringRow(table, "john", "doe", "ignored"); writeStringRow(table, "jane", "roe", "ignored"); maskDisplayWithFullName(identifier); @@ -4038,7 +4032,6 @@ void testColumnMaskingProjectionAcrossMultipleSplits() throws Exception { List values = new ArrayList<>(); for (InternalRow row : rows) { - // every split must be projected back to the query's arity assertThat(row.getFieldCount()).isEqualTo(1); values.add(row.getString(0).toString()); } @@ -4056,7 +4049,6 @@ void testColumnMaskingOnRowFilterColumnWithProjection() throws Exception { Collections.emptyMap()); writeStringRow(table, "john", "doe", "secret", "o1"); - // the filter pulls unprojected "display" into the read type, activating its mask LeafPredicate displayFilter = LeafPredicate.of( new FieldTransform(new FieldRef(2, "display", DataTypes.STRING())), @@ -4065,7 +4057,6 @@ void testColumnMaskingOnRowFilterColumnWithProjection() throws Exception { setRowFilter(identifier, Collections.singletonList(displayFilter)); maskDisplayWithFullName(identifier); - // project only "other": the mask target and inputs are all unprojected ReadBuilder readBuilder = table.newReadBuilder().withProjection(new int[] {3}); List splits = readBuilder.newScan().plan().splits(); List rows = @@ -4098,7 +4089,6 @@ void testColumnMaskingRevokedOnSameTableRead() throws Exception { assertThat(masked).hasSize(1); assertThat(masked.get(0).getString(0).toString()).isEqualTo("john-doe"); - // revoke the rules: the same TableRead must drop the widened read type setColumnMasking(identifier, new HashMap<>()); List plain = collectRows( @@ -4119,7 +4109,6 @@ void testColumnMaskingGrantedAfterReadSchemaFixed() throws Exception { Collections.emptyMap()); writeStringRow(table, "john", "doe", "plain"); - // first read without rules fixes the read schema to the projection ReadBuilder readBuilder = table.newReadBuilder().withProjection(new int[] {2}); TableRead read = readBuilder.newRead(); List plain = @@ -4166,7 +4155,6 @@ void testColumnMaskingPreservesNestedProjection() throws Exception { write.close(); commit.close(); - // a nested-pruned read type, as engines push down RowType tableRowType = table.rowType(); DataField sField = tableRowType.getField("s"); RowType prunedS = ((RowType) sField.type()).project("b"); @@ -4176,7 +4164,6 @@ void testColumnMaskingPreservesNestedProjection() throws Exception { tableRowType.getField("display"), new DataField(sField.id(), "s", prunedS))); - // sanity: the nested-pruned read works without masking ReadBuilder readBuilder = table.newReadBuilder().withReadType(prunedReadType); List rows = collectRows( @@ -4185,7 +4172,6 @@ void testColumnMaskingPreservesNestedProjection() throws Exception { assertThat(rows).hasSize(1); assertThat(rows.get(0).getRow(1, 1).getString(0).toString()).isEqualTo("BV"); - // mask "display" from unprojected "extra": widening must keep "s" pruned Map columnMasking = new HashMap<>(); columnMasking.put( "display", @@ -4216,7 +4202,6 @@ void testColumnMaskingStaleRuleFailsClosed() throws Exception { Collections.emptyMap()); writeStringRow(table, "john", "doe", "secret"); - // mask target absent from the schema (e.g. renamed since the rule was written) Map staleTarget = new HashMap<>(); staleTarget.put( "renamed_away", @@ -4226,7 +4211,6 @@ void testColumnMaskingStaleRuleFailsClosed() throws Exception { .isInstanceOf(IllegalArgumentException.class) .hasMessageContaining("does not exist in table schema"); - // mask input absent from the schema Map staleInput = new HashMap<>(); staleInput.put( "display", @@ -4250,7 +4234,6 @@ void testColumnMaskingRuleChangeOnRetainedColumn() throws Exception { Collections.emptyMap()); writeStringRow(table, "d1", "a1", "b1"); - // first rules widen and fix the read schema to [display, hidden_a] Map rules = new HashMap<>(); rules.put( "display", @@ -4268,7 +4251,6 @@ void testColumnMaskingRuleChangeOnRetainedColumn() throws Exception { table.rowType().project("display")); assertThat(masked.get(0).getString(0).toString()).isEqualTo("a1"); - // the new rules mask only hidden_a, retained but unread: must not activate rules.clear(); rules.put( "hidden_a", @@ -4313,7 +4295,6 @@ void testColumnMaskingRejectsNestedPrunedMaskTarget() throws Exception { write.close(); commit.close(); - // the mask TARGETS the struct column "s" (reading another column) RowType tableRowType = table.rowType(); DataField sField = tableRowType.getField("s"); Map masking = new HashMap<>(); @@ -4322,7 +4303,6 @@ void testColumnMaskingRejectsNestedPrunedMaskTarget() throws Exception { new ConcatTransform(Collections.singletonList(BinaryString.fromString("****")))); setColumnMasking(identifier, masking); - // projecting "s" nested-pruned would write the mask into a partial slot: fail closed RowType prunedS = ((RowType) sField.type()).project("b"); RowType prunedReadType = new RowType( @@ -4351,7 +4331,6 @@ void testColumnMaskingOnColumnAddedAfterSnapshot() throws Exception { identifier, stringFields("first", "display"), Collections.emptyMap()); writeStringRow(table, "john", "d1"); // snapshot 1 - // add a column, then mask it: the rule is valid only in the latest schema catalog.alterTable( identifier, Collections.singletonList(SchemaChange.addColumn("extra", DataTypes.STRING())), @@ -4362,7 +4341,6 @@ void testColumnMaskingOnColumnAddedAfterSnapshot() throws Exception { new ConcatTransform(Collections.singletonList(BinaryString.fromString("****")))); setColumnMasking(identifier, masking); - // the latest read masks the new column Table latest = catalog.getTable(identifier); ReadBuilder latestRead = latest.newReadBuilder(); List latestRows = @@ -4372,7 +4350,6 @@ void testColumnMaskingOnColumnAddedAfterSnapshot() throws Exception { assertThat(latestRows).hasSize(1); assertThat(latestRows.get(0).getString(2).toString()).isEqualTo("****"); - // a time-travel read of the old snapshot must not fail on the newer rule Table old = catalog.getTable(identifier) .copy(Collections.singletonMap(CoreOptions.SCAN_SNAPSHOT_ID.key(), "1")); @@ -4394,7 +4371,6 @@ void testColumnMaskingRenamedColumnTimeTravelFailsClosed() throws Exception { identifier, stringFields("first", "secret"), Collections.emptyMap()); writeStringRow(table, "john", "s1"); // snapshot 1, column named "secret" - // rename the column, then mask it under the new name catalog.alterTable( identifier, Collections.singletonList(SchemaChange.renameColumn("secret", "masked_secret")), @@ -4405,7 +4381,6 @@ void testColumnMaskingRenamedColumnTimeTravelFailsClosed() throws Exception { new ConcatTransform(Collections.singletonList(BinaryString.fromString("****")))); setColumnMasking(identifier, masking); - // the latest read masks the renamed column Table latest = catalog.getTable(identifier); ReadBuilder latestRead = latest.newReadBuilder(); List latestRows = @@ -4414,9 +4389,6 @@ void testColumnMaskingRenamedColumnTimeTravelFailsClosed() throws Exception { latest.rowType()); assertThat(latestRows.get(0).getString(1).toString()).isEqualTo("****"); - // a time-travel read of the pre-rename snapshot exposes the same physical column - // as "secret"; the rule keyed on "masked_secret" would be silently skipped by name - // and leak the raw value -- it must fail closed instead Table old = catalog.getTable(identifier) .copy(Collections.singletonMap(CoreOptions.SCAN_SNAPSHOT_ID.key(), "1")); @@ -4437,8 +4409,6 @@ void testColumnMaskingSystemTargetInertWhenUnprojected() throws Exception { Collections.singletonMap(CoreOptions.ROW_TRACKING_ENABLED.key(), "true")); writeStringRow(table, "d1", "o1"); - // a mask on a system column the query does not project must be inert, not reject - // the whole query at plan time Map masking = new HashMap<>(); masking.put( "_ROW_ID", @@ -4463,7 +4433,6 @@ void testColumnMaskingInertTargetWithRenamedInputTimeTravel() throws Exception { identifier, stringFields("first", "old_input"), Collections.emptyMap()); writeStringRow(table, "john", "in1"); // snapshot 1 - // rename the input, then add a target column masked from the renamed input catalog.alterTable( identifier, Collections.singletonList(SchemaChange.renameColumn("old_input", "renamed_input")), @@ -4478,8 +4447,6 @@ void testColumnMaskingInertTargetWithRenamedInputTimeTravel() throws Exception { new FieldTransform(new FieldRef(1, "renamed_input", DataTypes.STRING()))); setColumnMasking(identifier, masking); - // the pre-rename snapshot predates "display": the mask cannot output there, so the - // rename of its input must not fail the read Table old = catalog.getTable(identifier) .copy(Collections.singletonMap(CoreOptions.SCAN_SNAPSHOT_ID.key(), "1")); @@ -4503,7 +4470,6 @@ void testColumnMaskingRevalidatedAfterRulesDisappear() throws Exception { StreamTableScan scan = table.newReadBuilder().newStreamScan(); - // plan 1: a valid mask on "secret" is validated and cached Map masking = new HashMap<>(); masking.put( "secret", @@ -4511,12 +4477,9 @@ void testColumnMaskingRevalidatedAfterRulesDisappear() throws Exception { setColumnMasking(identifier, masking); scan.plan(); - // plan 2: rules disappear -- the cached validation must be forgotten setColumnMasking(identifier, Collections.emptyMap()); scan.plan(); - // the masked column is renamed away, then the identical rule is restored; a stale-rule - // cache short-circuit would skip re-validation and silently stop masking catalog.alterTable( identifier, Collections.singletonList(SchemaChange.renameColumn("secret", "hidden")), @@ -4545,8 +4508,6 @@ void testColumnMaskingRenamedUnderLiveScanFailsClosed() throws Exception { StreamTableScan scan = table.newReadBuilder().newStreamScan(); scan.plan(); - // the masked column is renamed while the rules stay identical: the live scan must - // notice on its next plan and fail closed, not keep planning on the stale rule catalog.alterTable( identifier, Collections.singletonList(SchemaChange.renameColumn("secret", "hidden")), @@ -4583,7 +4544,6 @@ void testColumnMaskingRejectsNestedPrunedRuleInput() throws Exception { write.close(); commit.close(); - // the mask on "display" reads the whole struct column "s" RowType tableRowType = table.rowType(); DataField sField = tableRowType.getField("s"); Map masking = new HashMap<>(); @@ -4592,7 +4552,6 @@ void testColumnMaskingRejectsNestedPrunedRuleInput() throws Exception { new CastTransform(new FieldRef(1, "s", sField.type()), DataTypes.STRING())); setColumnMasking(identifier, masking); - // projecting "s" nested-pruned would hand the mask a partial struct: fail closed RowType prunedS = ((RowType) sField.type()).project("b"); RowType prunedReadType = new RowType( @@ -4625,7 +4584,6 @@ void testColumnMaskingWithDataEvolutionColumnFiles() throws Exception { options.put(CoreOptions.DATA_EVOLUTION_ENABLED.key(), "true"); Table table = createMaskingAuthTable(identifier, fields, options); - // one row group split across two columnar files: (f0, f1) and (f2) RowType tableRowType = table.rowType(); BatchWriteBuilder builder = table.newBatchWriteBuilder(); try (BatchTableWrite write0 = @@ -4653,7 +4611,6 @@ void testColumnMaskingWithDataEvolutionColumnFiles() throws Exception { builder.newCommit().commit(commitables); } - // mask f1 from f2 and project only f1: the scan must keep f2's file Map masking = new HashMap<>(); masking.put( "f1", @@ -4701,7 +4658,6 @@ void testColumnMaskingRejectsUnprojectedBlobViewInput() throws Exception { write.close(); commit.close(); - // the mask reads the unprojected blob-view column: resolution cannot apply Map masking = new HashMap<>(); masking.put( "display", @@ -4736,7 +4692,6 @@ void testColumnMaskingDisablesTopNPushdown() throws Exception { fields, Collections.singletonMap( CoreOptions.SOURCE_SPLIT_TARGET_SIZE.key(), "1 b")); - // two splits with opposite raw/masked ordering for (int[] row : new int[][] {{100, 0}, {50, 1000}}) { BatchWriteBuilder writeBuilder = table.newBatchWriteBuilder(); BatchTableWrite write = writeBuilder.newWrite(); @@ -4747,7 +4702,6 @@ void testColumnMaskingDisablesTopNPushdown() throws Exception { commit.close(); } - // display := source inverts the ordering the raw statistics suggest Map masking = new HashMap<>(); masking.put("display", new FieldTransform(new FieldRef(1, "source", DataTypes.INT()))); setColumnMasking(identifier, masking); @@ -4761,7 +4715,6 @@ void testColumnMaskingDisablesTopNPushdown() throws Exception { collectRows( readBuilder.newRead().createReader(splits), table.rowType().project("display")); - // split pruning must not drop the split holding the real top row (1000) assertThat( rows.stream() .map(row -> row.getInt(0)) @@ -4788,7 +4741,6 @@ void testColumnMaskingReadingSystemField() throws Exception { write.close(); commit.close(); - // the mask reads the projected _ROW_ID metadata field: not a stale rule Map masking = new HashMap<>(); masking.put( "display", @@ -4806,7 +4758,6 @@ void testColumnMaskingReadingSystemField() throws Exception { readBuilder.newRead().createReader(readBuilder.newScan().plan().splits()), readType); assertThat(rows).hasSize(1); - // display masked to the row id assertThat(rows.get(0).getLong(0)).isEqualTo(rows.get(0).getLong(1)); } @@ -4821,7 +4772,6 @@ void testColumnMaskingSystemFieldValidation() throws Exception { Collections.singletonMap(CoreOptions.ROW_TRACKING_ENABLED.key(), "true")); writeStringRow(table, "d1", "o1"); - // a mask keyed by a key-reader-internal name is stale, not a system field Map masking = new HashMap<>(); masking.put( "_KEY_display", @@ -4831,7 +4781,6 @@ void testColumnMaskingSystemFieldValidation() throws Exception { .isInstanceOf(IllegalArgumentException.class) .hasMessageContaining("does not exist in table schema"); - // a rule reading an unprojected system field fails clearly at plan time masking.clear(); masking.put( "display", @@ -4856,7 +4805,6 @@ void testColumnMaskingDisablesFilterStatsPruning() throws Exception { fields, Collections.singletonMap( CoreOptions.SOURCE_SPLIT_TARGET_SIZE.key(), "1 b")); - // two splits whose raw and masked values order oppositely for (int[] row : new int[][] {{1, 1000}, {900, 10}}) { BatchWriteBuilder writeBuilder = table.newBatchWriteBuilder(); BatchTableWrite write = writeBuilder.newWrite(); @@ -4870,9 +4818,6 @@ void testColumnMaskingDisablesFilterStatsPruning() throws Exception { masking.put("amount", new FieldTransform(new FieldRef(1, "src", DataTypes.INT()))); setColumnMasking(identifier, masking); - // the predicate applies to masked values, with no engine help (no executeFilter): - // raw statistics must not prune the split whose masked value matches, and the - // raw-matching row must not come back LeafPredicate amountFilter = LeafPredicate.of( new FieldTransform(new FieldRef(0, "amount", DataTypes.INT())), @@ -4916,20 +4861,16 @@ void testMaskGrowthOnPushedFilterColumn() throws Exception { table.rowType().project("display")); assertThat(plain).hasSize(1); - // a mask now covers the filter column Map masking = new HashMap<>(); masking.put( "display", new ConcatTransform(Collections.singletonList(BinaryString.fromString("****")))); setColumnMasking(identifier, masking); - // this scan already pruned with the column's raw statistics: it must fail closed assertThatThrownBy(scan::plan) .isInstanceOf(IllegalStateException.class) .hasMessageContaining("Recreate the scan"); - // a fresh scan carries no raw pruning, and the existing reader evaluates the - // filter on the masked value: 'd1' no longer matches List splits = readBuilder.newScan().plan().splits(); List masked = collectRows(read.createReader(splits), table.rowType().project("display")); @@ -4961,7 +4902,6 @@ void testDeferredFilterAppliesToPartitionListing() throws Exception { new FieldTransform(new FieldRef(0, "p", DataTypes.STRING())), Equal.INSTANCE, Collections.singletonList(BinaryString.fromString("a"))); - // partition listing bypasses plan(): the filter must still prune assertThat( table.newReadBuilder() .withFilter(partitionFilter) @@ -4995,8 +4935,6 @@ void testColumnMaskingDisablesLimitPushdown() throws Exception { masking.put("amount", new FieldTransform(new FieldRef(1, "src", DataTypes.INT()))); setColumnMasking(identifier, masking); - // the whole filter sits on the masked column: nothing is pushed down, but limit - // pruning must still know a read-time filter drops rows LeafPredicate amountFilter = LeafPredicate.of( new FieldTransform(new FieldRef(0, "amount", DataTypes.INT())), @@ -5032,13 +4970,10 @@ void testFilterOnMaskedPartitionColumn() throws Exception { writeStringRow(table, "a", "va"); writeStringRow(table, "b", "vb"); - // the partition column is masked to another column's value Map masking = new HashMap<>(); masking.put("p", new FieldTransform(new FieldRef(1, "v", DataTypes.STRING()))); setColumnMasking(identifier, masking); - // engines consume partition filters without re-evaluating them, so the read - // itself must evaluate this predicate, on the masked value LeafPredicate maskedMatch = LeafPredicate.of( new FieldTransform(new FieldRef(0, "p", DataTypes.STRING())), @@ -5049,7 +4984,6 @@ void testFilterOnMaskedPartitionColumn() throws Exception { assertThat(rows.get(0).getString(0).toString()).isEqualTo("vb"); assertThat(rows.get(0).getString(1).toString()).isEqualTo("vb"); - // the raw partition value must not match: matching it would reveal the raw value LeafPredicate rawMatch = LeafPredicate.of( new FieldTransform(new FieldRef(0, "p", DataTypes.STRING())), @@ -5057,8 +4991,6 @@ void testFilterOnMaskedPartitionColumn() throws Exception { Collections.singletonList(BinaryString.fromString("a"))); assertThat(readWithFilter(table, rawMatch, "p", "v")).isEmpty(); - // filter column not projected: it is read and masked for the filter only, then - // projected back out List unprojected = readWithFilter(table, maskedMatch, "v"); assertThat(unprojected).hasSize(1); assertThat(unprojected.get(0).getString(0).toString()).isEqualTo("vb"); @@ -5081,8 +5013,6 @@ void testLimitWithFilterOnMaskedPartitionColumn() throws Exception { masking.put("p", new FieldTransform(new FieldRef(1, "v", DataTypes.STRING()))); setColumnMasking(identifier, masking); - // the filter is partition-only but sits on a masked column, so it drops rows at - // read time: limit pruning must not pick splits by raw row counts LeafPredicate maskedMatch = LeafPredicate.of( new FieldTransform(new FieldRef(0, "p", DataTypes.STRING())), @@ -5127,8 +5057,6 @@ void testMaskedPkFilterNotAppliedOnRawValues() throws Exception { masking.put("id", new FieldTransform(new FieldRef(1, "src", DataTypes.INT()))); setColumnMasking(identifier, masking); - // the key filter matches a masked value only: key-range skipping inside the - // merge read must not drop the row by its raw key LeafPredicate idFilter = LeafPredicate.of( new FieldTransform(new FieldRef(0, "id", DataTypes.INT())), @@ -5624,16 +5552,12 @@ void testColumnMaskingOrFilterWithUnprojectedOperand() throws Exception { BinaryString.fromString("bee"), BinaryString.fromString("cee"))); - // mask a -> "MASKED"; b and c are not masked Map masking = new HashMap<>(); masking.put( "a", new ConcatTransform(Collections.singletonList(BinaryString.fromString("MASKED")))); setColumnMasking(identifier, masking); - // WHERE a = 'MASKED' OR b = 'bee' -- one masked operand, one plain, neither projected. - // splitAnd does not split the OR, so retainFields keeps the whole disjunction, but only - // the masked column 'a' is widened in; 'b' is missing from the read schema. Predicate onMasked = LeafPredicate.of( new FieldTransform(new FieldRef(0, "a", DataTypes.STRING())), @@ -5689,8 +5613,6 @@ void testColumnMaskingPartitionListingBeforePlan() throws Exception { Equal.INSTANCE, Collections.singletonList(BinaryString.fromString("MASKED"))); - // listPartitionEntries() bypasses plan(); a later plan() on the same scan must not treat - // the masks discovered then as a rule change against an already-pushed filter. InnerTableScan scan = (InnerTableScan) table.newReadBuilder().withFilter(onMasked).newScan(); assertThat(scan.listPartitionEntries()).hasSize(1); @@ -5714,15 +5636,11 @@ void testQueryAuthLimitDoesNotCutRowsBeforeFilter() throws Exception { ""), true); Table table = catalog.getTable(identifier); - // one file, the matching row second commitRows( table, GenericRow.of(BinaryString.fromString("no"), BinaryString.fromString("r1")), GenericRow.of(BinaryString.fromString("yes"), BinaryString.fromString("r2"))); - // The reader does not evaluate the query filter on an auth-enabled table; engines - // re-apply it. A read-level limit would cap the raw rows at "no" and the engine would - // then filter it away, losing the matching row entirely. Predicate onA = LeafPredicate.of( new FieldTransform(new FieldRef(0, "a", DataTypes.STRING())), @@ -5760,15 +5678,12 @@ void testMaskedPartitionKeyRejectsPartitionFilter() throws Exception { Equal.INSTANCE, Collections.singletonList(BinaryString.fromString("vb"))); - // routed through withPartitionFilter (as Spark does) the predicate never reaches the - // masked value, so it must fail closed rather than prune on the raw partition value InnerTableScan partitionScan = (InnerTableScan) table.newReadBuilder().newScan(); partitionScan.withPartitionFilter(maskedMatch); assertThatThrownBy(partitionScan::plan) .isInstanceOf(UnsupportedOperationException.class) .hasMessageContaining("masked partition key"); - // the same predicate through withFilter is evaluated post-mask and still works assertThat(readWithFilter(table, maskedMatch, "p", "v")).hasSize(1); } @@ -5784,12 +5699,10 @@ void testPartitionFilterAllowedOnUnmaskedPartitionKey() throws Exception { Collections.emptyMap()); writeStringRows(table, new String[] {"x", "a", "v1"}, new String[] {"y", "b", "v2"}); - // only p2 is masked Map masking = new HashMap<>(); masking.put("p2", new FieldTransform(new FieldRef(2, "v", DataTypes.STRING()))); setColumnMasking(identifier, masking); - // a partition predicate touching only the UNMASKED key stays prunable LeafPredicate onP1 = LeafPredicate.of( new FieldTransform(new FieldRef(0, "p1", DataTypes.STRING())), @@ -5799,7 +5712,6 @@ void testPartitionFilterAllowedOnUnmaskedPartitionKey() throws Exception { okScan.withPartitionFilter(onP1); assertThat(okScan.plan().splits()).isNotEmpty(); - // touching the masked key is still rejected LeafPredicate onP2 = LeafPredicate.of( new FieldTransform(new FieldRef(1, "p2", DataTypes.STRING())), @@ -5824,8 +5736,6 @@ void testExplicitPartitionFilterSurvivesDeferredQueryFilter() throws Exception { Collections.emptyMap()); writeStringRows(table, new String[] {"a", "va"}, new String[] {"b", "vb"}); - // no masking: query auth alone defers the filter push, which must not overwrite - // the caller's partition filter LeafPredicate pAtLeastA = LeafPredicate.of( new FieldTransform(new FieldRef(0, "p", DataTypes.STRING())), @@ -5833,7 +5743,6 @@ void testExplicitPartitionFilterSurvivesDeferredQueryFilter() throws Exception { Collections.singletonList(BinaryString.fromString("a"))); assertThat(readPartitionB(table, pAtLeastA)).containsExactly("b"); - // control: without query auth the filter is pushed eagerly, so this always held Identifier plain = Identifier.create("test_table_db", "plain_part_filter_with_filter"); catalog.createTable( plain, @@ -5886,8 +5795,6 @@ void testQueryAuthLimitAppliesWhenReaderExecutesFilter() throws Exception { GenericRow.of(BinaryString.fromString("yes"), BinaryString.fromString("r2")), GenericRow.of(BinaryString.fromString("yes"), BinaryString.fromString("r3"))); - // executeFilter() means the reader evaluates the predicate itself, so capping after it - // is safe and the caller's limit must still be honoured Predicate onA = LeafPredicate.of( new FieldTransform(new FieldRef(0, "a", DataTypes.STRING())), @@ -5906,8 +5813,8 @@ void testQueryAuthLimitAppliesWhenReaderExecutesFilter() throws Exception { } @Test - void testPartitionFilterFieldsReplacedOnRepush() throws Exception { - Identifier identifier = Identifier.create("test_table_db", "auth_part_filter_repush"); + void testPartitionFilterFieldsReplacedOnReapply() throws Exception { + Identifier identifier = Identifier.create("test_table_db", "auth_part_filter_reapply"); Table table = createMaskingAuthTable( identifier, @@ -5932,8 +5839,6 @@ void testPartitionFilterFieldsReplacedOnRepush() throws Exception { Equal.INSTANCE, Collections.singletonList(BinaryString.fromString("x"))); - // the second push overwrites the first in ManifestsReader, so the tracked fields must be - // replaced too: the effective predicate only touches the unmasked key InnerTableScan scan = (InnerTableScan) table.newReadBuilder().newScan(); scan.withPartitionFilter(onMaskedP2); scan.withPartitionFilter(onPlainP1); @@ -5953,7 +5858,6 @@ void testPhysicalMetadataSystemTablesRejectedUnderQueryAuth() throws Exception { new ConcatTransform(Collections.singletonList(BinaryString.fromString("****")))); setColumnMasking(identifier, masking); - // these report per-column min/max of the raw files, which no mask can cover for (String suffix : Arrays.asList("files", "file_key_ranges", "binlog")) { Identifier sysId = Identifier.create( @@ -5964,7 +5868,6 @@ void testPhysicalMetadataSystemTablesRejectedUnderQueryAuth() throws Exception { .hasMessageContaining("query-auth table"); } - // the row-producing ones read through the masking reader and stay available for (String suffix : Arrays.asList("audit_log", "ro")) { Identifier sysId = Identifier.create( @@ -5984,8 +5887,6 @@ void testMaskReadingAnotherMaskedColumnRejected() throws Exception { identifier, stringFields("secret", "display"), Collections.emptyMap()); writeStringRow(table, "TOPSECRET", "ignored"); - // display := secret, while secret is masked. A transform reads the raw row, so this - // would publish secret's raw value through display. Map compose = new HashMap<>(); compose.put( "secret", @@ -5996,7 +5897,6 @@ void testMaskReadingAnotherMaskedColumnRejected() throws Exception { .isInstanceOf(IllegalArgumentException.class) .hasMessageContaining("which is masked too"); - // a mask reading an unmasked column, and one reading its own column, both stay valid Map plain = new HashMap<>(); plain.put("display", new FieldTransform(new FieldRef(0, "secret", DataTypes.STRING()))); setColumnMasking(identifier, plain); @@ -6013,8 +5913,6 @@ void testMaskReadingAnotherMaskedColumnRejected() throws Exception { @Test void testQueryAuthRejectedWhereItCannotBeEnforced() throws Exception { - // a non-file-store table never reads through the auth reader, so accepting the option - // would leave the rules silently inert for (String type : Arrays.asList("format-table", "object-table")) { Map opts = new HashMap<>(); opts.put(QUERY_AUTH_ENABLED.key(), "true"); @@ -6037,7 +5935,6 @@ void testQueryAuthRejectedWhereItCannotBeEnforced() throws Exception { .hasMessageContaining(QUERY_AUTH_ENABLED.key()); } - // search ranks raw index values, so it is refused rather than answered from them Identifier identifier = Identifier.create("test_table_db", "auth_search_rejected"); Table table = createMaskingAuthTable( @@ -6047,7 +5944,6 @@ void testQueryAuthRejectedWhereItCannotBeEnforced() throws Exception { .isInstanceOf(UnsupportedOperationException.class) .hasMessageContaining("query-auth table"); - // the lookup cache serves rows straight from the store assertThatThrownBy(() -> new LocalTableQuery((FileStoreTable) table)) .isInstanceOf(UnsupportedOperationException.class) .hasMessageContaining("query-auth table"); @@ -6071,8 +5967,6 @@ void testRowIdFilterOnDataEvolutionQueryAuthTable() throws Exception { builder.newCommit().commit(write.prepareCommit()); } - // no masking rules at all -- a _ROW_ID predicate must still plan. Data-evolution - // statistics carry only logical columns, so the row-id part must not be pushed. Predicate onRowId = LeafPredicate.of( new FieldTransform( diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/RESTCatalogITCase.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/RESTCatalogITCase.java index d1276136ee84..6e606d1a2d7f 100644 --- a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/RESTCatalogITCase.java +++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/RESTCatalogITCase.java @@ -373,7 +373,6 @@ public void testColumnMaskingCrossColumnWithProjection() { "CREATE TABLE %s.%s (first_name STRING, last_name STRING, display STRING, other_col STRING)" + " WITH ('query-auth.enabled' = 'true', 'source.split.target-size' = '1 b')", DATABASE_NAME, maskingTable)); - // two commits so the scan yields multiple splits batchSql( String.format( "INSERT INTO %s.%s VALUES ('john', 'doe', 'ignored', 'o1')", @@ -383,7 +382,6 @@ public void testColumnMaskingCrossColumnWithProjection() { "INSERT INTO %s.%s VALUES ('jane', 'roe', 'ignored', 'o2')", DATABASE_NAME, maskingTable)); - // the mask on "display" reads OTHER columns: concat_ws('-', first_name, last_name) Map columnMasking = new HashMap<>(); columnMasking.put( "display", @@ -395,13 +393,11 @@ public void testColumnMaskingCrossColumnWithProjection() { restCatalogServer.setColumnMaskingAuth( Identifier.create(DATABASE_NAME, maskingTable), columnMasking); - // project only the masked target: its input columns must be read regardless assertThat( batchSql( String.format( "SELECT display FROM %s.%s", DATABASE_NAME, maskingTable))) .containsExactlyInAnyOrder(Row.of("john-doe"), Row.of("jane-roe")); - // a projection without the masked column is unaffected assertThat( batchSql( String.format( @@ -428,22 +424,18 @@ public void testFilterOnMaskedPartitionColumn() { restCatalogServer.setColumnMaskingAuth( Identifier.create(DATABASE_NAME, maskingTable), columnMasking); - // Flink consumes bounded partition filters without re-evaluating them, so the - // source itself must evaluate the predicate, on the masked value assertThat( batchSql( String.format( "SELECT p, v FROM %s.%s WHERE p = 'vb'", DATABASE_NAME, maskingTable))) .containsExactlyInAnyOrder(Row.of("vb", "vb")); - // the raw partition value must not match assertThat( batchSql( String.format( "SELECT p, v FROM %s.%s WHERE p = 'a'", DATABASE_NAME, maskingTable))) .isEmpty(); - // filter column not projected assertThat( batchSql( String.format( diff --git a/paimon-spark/paimon-spark-ut/src/test/java/org/apache/paimon/spark/SparkCatalogWithRestTest.java b/paimon-spark/paimon-spark-ut/src/test/java/org/apache/paimon/spark/SparkCatalogWithRestTest.java index 8eaf387b9a49..c9ff2b5fd34a 100644 --- a/paimon-spark/paimon-spark-ut/src/test/java/org/apache/paimon/spark/SparkCatalogWithRestTest.java +++ b/paimon-spark/paimon-spark-ut/src/test/java/org/apache/paimon/spark/SparkCatalogWithRestTest.java @@ -369,11 +369,9 @@ public void testColumnMaskingCrossColumnWithProjection() { spark.sql( "CREATE TABLE t_cross_column_masking (first_name STRING, last_name STRING, display STRING, other_col STRING)" + " TBLPROPERTIES ('query-auth.enabled'='true', 'source.split.target-size'='1 b')"); - // two commits so the scan yields multiple splits spark.sql("INSERT INTO t_cross_column_masking VALUES ('john', 'doe', 'ignored', 'o1')"); spark.sql("INSERT INTO t_cross_column_masking VALUES ('jane', 'roe', 'ignored', 'o2')"); - // the mask on "display" reads OTHER columns: concat_ws('-', first_name, last_name) Map columnMasking = new HashMap<>(); columnMasking.put( "display", @@ -385,13 +383,11 @@ public void testColumnMaskingCrossColumnWithProjection() { restCatalogServer.setColumnMaskingAuth( Identifier.create("db2", "t_cross_column_masking"), columnMasking); - // project only the masked target: its input columns must be read regardless assertThat( spark.sql("SELECT display FROM t_cross_column_masking ORDER BY other_col") .collectAsList() .toString()) .isEqualTo("[[john-doe], [jane-roe]]"); - // a projection without the masked column is unaffected assertThat( spark.sql("SELECT other_col FROM t_cross_column_masking ORDER BY other_col") .collectAsList() @@ -414,8 +410,6 @@ public void testRowFilterDisablesAggregatePushdown() { restCatalogServer.setRowFilterAuth( Identifier.create("db2", "t_agg_pushdown"), Collections.singletonList(idFilter)); - // statistics-based aggregate pushdown must not bypass the read-time row filter - // (today it degrades because auth splits are not DataSplits; this anchors that) assertThat(spark.sql("SELECT COUNT(*) FROM t_agg_pushdown").collectAsList().toString()) .isEqualTo("[[2]]"); } From 694cf4cc9c8a631123555f2a571f394dc66fc551 Mon Sep 17 00:00:00 2001 From: Li Jiajia Date: Fri, 31 Jul 2026 12:54:21 -0400 Subject: [PATCH 6/7] [core] Close the remaining fail-open paths around query auth listPartitionEntries fetched the auth result but used only its masking rules, so a row filter on a partition key never reached pruning: the listing reported every partition, with its file and record counts, including the ones the filter excludes. plan() has always applied it; this path bypasses plan(). Four more paths served raw values with the rules in place: - t$statistics serialises the merged row count and per-column min/max, distinct and null counts, but was not rejected alongside t$files; - the table-type check ran at create only, so ALTER could turn query-auth.enabled on for a format or object table whose read ignores it. It now lives in schema validation, which both paths go through; - the search guard sat on the scan factory alone, while a pre-built plan reaches newVectorRead, newBatchVectorRead and newFullTextRead directly. The Flink and Spark subclasses override those, so they are guarded too; - a mask on _ROW_ID makes the predicate carry masked ids, which RowIdPredicateVisitor turned into a raw row range, pruning away the files the query matches. The rules are not known when the filter arrives, so the extraction is skipped whenever query auth is on. Restores the null check on DataEvolutionBatchScan's table, which the tests that exercise withFilter in isolation rely on. Two test gaps closed as well: nothing failed when DataTableStreamScan's removed second filter push was put back, and both new cases in MergeFileSplitReadTest kept every assertion inside the per-row loop, so an empty read would have passed them silently. --- .../globalindex/DataEvolutionBatchScan.java | 34 +++- .../paimon/schema/SchemaValidation.java | 13 ++ .../table/source/AbstractBatchTableScan.java | 9 +- .../table/source/AbstractDataTableScan.java | 2 +- .../source/BatchVectorSearchBuilderImpl.java | 1 + .../source/FullTextSearchBuilderImpl.java | 1 + .../table/source/PrimaryKeyBatchScan.java | 5 + .../table/source/VectorSearchBuilderImpl.java | 3 +- .../table/system/SystemTableLoader.java | 6 +- .../operation/MergeFileSplitReadTest.java | 8 + .../apache/paimon/rest/RESTCatalogTest.java | 179 +++++++++++++++++- .../paimon/schema/SchemaValidationTest.java | 29 +++ .../FlinkVectorSearchBuilderImpl.java | 1 + .../read/SparkVectorSearchBuilderImpl.java | 1 + 14 files changed, 274 insertions(+), 18 deletions(-) diff --git a/paimon-core/src/main/java/org/apache/paimon/globalindex/DataEvolutionBatchScan.java b/paimon-core/src/main/java/org/apache/paimon/globalindex/DataEvolutionBatchScan.java index b76abfc6f88d..589a9980e294 100644 --- a/paimon-core/src/main/java/org/apache/paimon/globalindex/DataEvolutionBatchScan.java +++ b/paimon-core/src/main/java/org/apache/paimon/globalindex/DataEvolutionBatchScan.java @@ -70,6 +70,8 @@ public class DataEvolutionBatchScan implements DataTableScan { private Predicate filter; private TopN topN; private Integer pushDownLimit; + // set when part of the filter reaches the reader only, so limit/TopN must not prune ahead of it + private boolean rowIdFilterDeferred; private RowRangeIndex pushedRowRangeIndex; private GlobalIndexResult globalIndexResult; @@ -90,9 +92,13 @@ public InnerTableScan withFilter(Predicate predicate) { return this; } - Optional> rowRanges = predicate.visit(new RowIdPredicateVisitor()); - if (rowRanges.isPresent()) { - withRowRanges(rowRanges.get()); + // a mask on _ROW_ID makes the predicate's ids the masked ones, so they must not become + // a raw row range; the rules are not known yet, so skip the extraction altogether + if (!queryAuthEnabled()) { + Optional> rowRanges = predicate.visit(new RowIdPredicateVisitor()); + if (rowRanges.isPresent()) { + withRowRanges(rowRanges.get()); + } } this.filter = predicate; @@ -100,6 +106,9 @@ public InnerTableScan withFilter(Predicate predicate) { // the wrapped scan defers the filter but strips only masked columns; row ids must // go here, since data-evolution statistics carry logical columns only Predicate residual = rowIdSafeResidualFilter(predicate); + // what is left out reaches the reader only, so the wrapped scan never learns a + // filter exists and would let limit/TopN prune ahead of it + rowIdFilterDeferred = containsRowId(predicate); if (residual != null) { batchScan.withFilter(residual); } @@ -177,8 +186,8 @@ public InnerTableScan withMetricRegistry(MetricRegistry metricsRegistry) { @Override public InnerTableScan withLimit(int limit) { + // forwarded in plan(), once withFilter has said whether a row-id part was deferred this.pushDownLimit = limit; - batchScan.withLimit(limit); return this; } @@ -288,7 +297,11 @@ public Plan plan() { } } - if (!globalIndexTopNCandidatesFound && topN != null) { + if (pushDownLimit != null && !rowIdFilterDeferred) { + batchScan.withLimit(pushDownLimit); + } + + if (!globalIndexTopNCandidatesFound && topN != null && !rowIdFilterDeferred) { batchScan.withTopN(topN); } @@ -301,21 +314,24 @@ public Plan plan() { } private boolean queryAuthEnabled() { + // the table is absent in tests that exercise withFilter in isolation CoreOptions options = table == null ? null : table.coreOptions(); return options != null && options.queryAuthEnabled(); } private Optional evalGlobalIndex() { + // the index ranks raw values, which a mask may invalidate; fall back to a full scan. + // Checked before the supplied result too: withGlobalIndexResult is public, so a caller + // can hand in one that was computed off the raw values. + if (queryAuthEnabled()) { + return Optional.empty(); + } if (this.globalIndexResult != null) { return Optional.of(globalIndexResult); } if (filter == null) { return Optional.empty(); } - if (queryAuthEnabled()) { - // the index ranks raw values, which a mask may invalidate; fall back to a full scan - return Optional.empty(); - } CoreOptions options = table.coreOptions(); if (!options.globalIndexEnabled()) { return Optional.empty(); diff --git a/paimon-core/src/main/java/org/apache/paimon/schema/SchemaValidation.java b/paimon-core/src/main/java/org/apache/paimon/schema/SchemaValidation.java index cc0ee88ad702..0139b059d9d9 100644 --- a/paimon-core/src/main/java/org/apache/paimon/schema/SchemaValidation.java +++ b/paimon-core/src/main/java/org/apache/paimon/schema/SchemaValidation.java @@ -156,6 +156,19 @@ public static void validateTableSchema(TableSchema schema, Set dynamicOp validateOnlyContainPrimitiveType(schema.fields(), schema.primaryKeys(), "primary key"); validateOnlyContainPrimitiveType(schema.fields(), schema.partitionKeys(), "partition"); + // only a file-store table reads through the auth reader; reject here rather than only at + // create time, so ALTER cannot turn the option on for a table type that ignores it + TableType tableType = options.type(); + if (options.queryAuthEnabled() + && tableType != TableType.TABLE + && tableType != TableType.MATERIALIZED_TABLE) { + throw new RuntimeException( + String.format( + "%s is not supported on a %s: its read does not apply row filters or " + + "column masks.", + CoreOptions.QUERY_AUTH_ENABLED.key(), tableType)); + } + if (options.primaryKeyNullable() && schema.primaryKeys().isEmpty()) { throw new IllegalArgumentException( String.format( diff --git a/paimon-core/src/main/java/org/apache/paimon/table/source/AbstractBatchTableScan.java b/paimon-core/src/main/java/org/apache/paimon/table/source/AbstractBatchTableScan.java index b4d927819a4e..8a3cd61ee979 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/source/AbstractBatchTableScan.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/source/AbstractBatchTableScan.java @@ -42,6 +42,7 @@ import java.time.Duration; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.Optional; import java.util.OptionalLong; @@ -152,12 +153,14 @@ protected Plan postProcessPlan(Plan plan) { @Override public List listPartitionEntries() { - // partition listing bypasses plan(), so resolve the masks here too: pushing a filter on a - // masked column against raw partition values would drop partitions the query matches + // partition listing bypasses plan(), so apply the rules here too: without the row filter + // it would report partitions the caller cannot read, and pushing a filter on a masked + // column against raw partition values would drop partitions the query matches TableQueryAuthResult authResult = authQuery(); + applyAuthFilter(authResult == null ? null : authResult.extractPredicate()); this.authMaskedFields = authResult == null - ? java.util.Collections.emptySet() + ? Collections.emptySet() : authResult.extractColumnMasking().keySet(); rejectMaskedPartitionFilter(); ensureFilterPushdown(authMaskedFields); diff --git a/paimon-core/src/main/java/org/apache/paimon/table/source/AbstractDataTableScan.java b/paimon-core/src/main/java/org/apache/paimon/table/source/AbstractDataTableScan.java index c35e27828230..95507f411975 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/source/AbstractDataTableScan.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/source/AbstractDataTableScan.java @@ -145,7 +145,7 @@ public final TableScan.Plan plan() { protected abstract TableScan.Plan planWithoutAuth(); - private void applyAuthFilter(@Nullable Predicate authPredicate) { + protected void applyAuthFilter(@Nullable Predicate authPredicate) { if (Objects.equals(authPredicate, appliedAuthPredicate)) { return; } diff --git a/paimon-core/src/main/java/org/apache/paimon/table/source/BatchVectorSearchBuilderImpl.java b/paimon-core/src/main/java/org/apache/paimon/table/source/BatchVectorSearchBuilderImpl.java index f47a8959df32..f9b83eeab470 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/source/BatchVectorSearchBuilderImpl.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/source/BatchVectorSearchBuilderImpl.java @@ -139,6 +139,7 @@ public VectorScan newVectorScan() { @Override public BatchVectorRead newBatchVectorRead() { + rejectUnderQueryAuth(); checkArgument(limit > 0, "Limit must be positive, set via withLimit()"); checkNotNull(vectorColumn, "Vector column must be set via withVectorColumn()"); checkArgument( diff --git a/paimon-core/src/main/java/org/apache/paimon/table/source/FullTextSearchBuilderImpl.java b/paimon-core/src/main/java/org/apache/paimon/table/source/FullTextSearchBuilderImpl.java index a692aec1eed0..27bcaa5a9270 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/source/FullTextSearchBuilderImpl.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/source/FullTextSearchBuilderImpl.java @@ -87,6 +87,7 @@ public FullTextScan newFullTextScan() { @Override public FullTextRead newFullTextRead() { + rejectUnderQueryAuth(); checkArgument(limit > 0, "Limit must be positive, set via withLimit()"); DataField textColumn = textColumn(); Optional definition = primaryKeyFullTextDefinition(textColumn); diff --git a/paimon-core/src/main/java/org/apache/paimon/table/source/PrimaryKeyBatchScan.java b/paimon-core/src/main/java/org/apache/paimon/table/source/PrimaryKeyBatchScan.java index b50f5dfcb5c6..fa4037f63306 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/source/PrimaryKeyBatchScan.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/source/PrimaryKeyBatchScan.java @@ -117,6 +117,11 @@ protected Plan preProcessPlan() { if (globalIndexSplitResult == null) { return null; } + if (options().queryAuthEnabled()) { + // the splits were selected from raw index values, which a mask may invalidate; + // plan normally instead, as the index evaluation below already does + return null; + } if (globalIndexSplitResult.snapshotId() > 0) { maybeCreateReadProtectionTag(globalIndexSplitResult.snapshotId()); } diff --git a/paimon-core/src/main/java/org/apache/paimon/table/source/VectorSearchBuilderImpl.java b/paimon-core/src/main/java/org/apache/paimon/table/source/VectorSearchBuilderImpl.java index 0bda59da80ac..67839191a365 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/source/VectorSearchBuilderImpl.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/source/VectorSearchBuilderImpl.java @@ -144,6 +144,7 @@ public VectorScan newVectorScan() { @Override public VectorRead newVectorRead() { + rejectUnderQueryAuth(); checkNotNull(vector, "vector must be set via withVector()"); if (isPrimaryKeyVectorSearch()) { return new PrimaryKeyVectorRead(table, vectorColumn, vector, limit, options, filter); @@ -162,7 +163,7 @@ public VectorSearchBuilderImpl withSnapshot(Snapshot snapshot) { return this; } - private void rejectUnderQueryAuth() { + protected void rejectUnderQueryAuth() { if (table.coreOptions().queryAuthEnabled()) { throw new UnsupportedOperationException( "Search is not supported on a query-auth table: the index ranks raw values, " diff --git a/paimon-core/src/main/java/org/apache/paimon/table/system/SystemTableLoader.java b/paimon-core/src/main/java/org/apache/paimon/table/system/SystemTableLoader.java index 615e078aa1ba..776730946850 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/system/SystemTableLoader.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/system/SystemTableLoader.java @@ -95,11 +95,11 @@ public class SystemTableLoader { Arrays.asList(ALL_TABLES, ALL_PARTITIONS, ALL_TABLE_OPTIONS, CATALOG_OPTIONS); /** - * System tables built from raw DataSplit metadata -- file names, row counts, per-column min/max - * -- none of which a column mask covers. + * System tables built from raw metadata -- file names, row counts, per-column min/max and + * distinct/null counts -- none of which a column mask covers. */ private static final List PHYSICAL_METADATA_TABLES = - Arrays.asList(FILES, FILE_KEY_RANGES, BINLOG); + Arrays.asList(FILES, FILE_KEY_RANGES, BINLOG, STATISTICS); @Nullable public static Table load(String type, FileStoreTable dataTable) { diff --git a/paimon-core/src/test/java/org/apache/paimon/operation/MergeFileSplitReadTest.java b/paimon-core/src/test/java/org/apache/paimon/operation/MergeFileSplitReadTest.java index cdabdaf1d80c..8e6a8541e0a5 100644 --- a/paimon-core/src/test/java/org/apache/paimon/operation/MergeFileSplitReadTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/operation/MergeFileSplitReadTest.java @@ -349,6 +349,8 @@ public void testRepeatedReadTypeResetsOuterProjection() throws Exception { scan.withSnapshot(snapshotId).plan().files().stream() .collect(Collectors.groupingBy(ManifestEntry::partition)); + int rowsRead = 0; + MergeFileSplitRead read = store.newRead(); read.withReadType(TestKeyValueGenerator.DEFAULT_ROW_TYPE.project("shopId", "dt", "hr")); read.withReadType( @@ -370,9 +372,11 @@ public void testRepeatedReadTypeResetsOuterProjection() throws Exception { RecordReaderIterator iterator = new RecordReaderIterator<>(reader); while (iterator.hasNext()) { assertThat(iterator.next().value().getFieldCount()).isEqualTo(4); + rowsRead++; } iterator.close(); } + assertThat(rowsRead).isPositive(); } @Test @@ -476,6 +480,8 @@ public void testIncrementalDiffReadOnProjectedMergeRead() throws Exception { SplitRead diffRead = new IncrementalDiffSplitRead(mergeRead); diffRead.withReadType(projection); + int rowsRead = 0; + for (Map.Entry> entry : filesByPartition.entrySet()) { List files = entry.getValue().stream().map(ManifestEntry::file).collect(Collectors.toList()); @@ -498,9 +504,11 @@ public void testIncrementalDiffReadOnProjectedMergeRead() throws Exception { assertThat(row.getString(1).toString()).hasSize(8); row.getInt(0); row.getInt(2); + rowsRead++; } iterator.close(); } + assertThat(rowsRead).isPositive(); } private List writeThenRead( diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTest.java b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTest.java index 4c3f77f77cf4..24ed7e285327 100644 --- a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTest.java @@ -42,9 +42,11 @@ import org.apache.paimon.function.Function; import org.apache.paimon.function.FunctionChange; import org.apache.paimon.function.FunctionDefinition; +import org.apache.paimon.globalindex.GlobalIndexResult; import org.apache.paimon.io.CompactIncrement; import org.apache.paimon.io.DataFileMeta; import org.apache.paimon.io.DataIncrement; +import org.apache.paimon.manifest.PartitionEntry; import org.apache.paimon.operation.BaseAppendFileStoreWrite; import org.apache.paimon.operation.FileStoreWrite; import org.apache.paimon.options.Options; @@ -104,6 +106,7 @@ import org.apache.paimon.types.DataTypes; import org.apache.paimon.types.RowType; import org.apache.paimon.utils.InternalRowUtils; +import org.apache.paimon.utils.Range; import org.apache.paimon.utils.SnapshotManager; import org.apache.paimon.utils.SnapshotNotExistException; import org.apache.paimon.utils.StringUtils; @@ -4837,6 +4840,51 @@ void testColumnMaskingDisablesFilterStatsPruning() throws Exception { .containsExactly(1000); } + @Test + void testStreamScanFilterOnMaskedColumnNotPushedToStats() throws Exception { + Identifier identifier = + Identifier.create("test_table_db", "auth_stream_masking_filter_stats"); + List fields = new ArrayList<>(); + fields.add(new DataField(0, "amount", DataTypes.INT())); + fields.add(new DataField(1, "src", DataTypes.INT())); + Table table = + createMaskingAuthTable( + identifier, + fields, + Collections.singletonMap( + CoreOptions.SOURCE_SPLIT_TARGET_SIZE.key(), "1 b")); + for (int[] row : new int[][] {{1, 1000}, {900, 10}}) { + BatchWriteBuilder writeBuilder = table.newBatchWriteBuilder(); + BatchTableWrite write = writeBuilder.newWrite(); + write.write(GenericRow.of(row[0], row[1])); + BatchTableCommit commit = writeBuilder.newCommit(); + commit.commit(write.prepareCommit()); + write.close(); + commit.close(); + } + Map masking = new HashMap<>(); + masking.put("amount", new FieldTransform(new FieldRef(1, "src", DataTypes.INT()))); + setColumnMasking(identifier, masking); + + LeafPredicate amountFilter = + LeafPredicate.of( + new FieldTransform(new FieldRef(0, "amount", DataTypes.INT())), + GreaterThan.INSTANCE, + Collections.singletonList(500)); + ReadBuilder readBuilder = + table.newReadBuilder().withProjection(new int[] {0}).withFilter(amountFilter); + StreamTableScan scan = readBuilder.newStreamScan(); + List rows = + collectRows( + readBuilder.newRead().createReader(scan.plan().splits()), + table.rowType().project("amount")); + assertThat( + rows.stream() + .map(row -> row.getInt(0)) + .collect(java.util.stream.Collectors.toList())) + .containsExactly(1000); + } + @Test void testMaskGrowthOnPushedFilterColumn() throws Exception { Identifier identifier = @@ -4910,6 +4958,32 @@ void testDeferredFilterAppliesToPartitionListing() throws Exception { .hasSize(1); } + @Test + void testRowFilterAppliesToPartitionListing() throws Exception { + Identifier identifier = + Identifier.create("test_table_db", "auth_row_filter_partition_listing"); + Table table = + createMaskingAuthTable( + identifier, + stringFields("p", "v"), + Collections.singletonList("p"), + Collections.emptyList(), + Collections.emptyMap()); + writeStringRows(table, new String[] {"a", "v1"}, new String[] {"b", "v2"}); + + LeafPredicate onA = + LeafPredicate.of( + new FieldTransform(new FieldRef(0, "p", DataTypes.STRING())), + Equal.INSTANCE, + Collections.singletonList(BinaryString.fromString("a"))); + setRowFilter(identifier, Collections.singletonList(onA)); + + List entries = + catalog.getTable(identifier).newReadBuilder().newScan().listPartitionEntries(); + assertThat(entries).hasSize(1); + assertThat(entries.get(0).partition().getString(0).toString()).isEqualTo("a"); + } + @Test void testColumnMaskingDisablesLimitPushdown() throws Exception { Identifier identifier = Identifier.create("test_table_db", "auth_table_masking_limit"); @@ -5858,7 +5932,7 @@ void testPhysicalMetadataSystemTablesRejectedUnderQueryAuth() throws Exception { new ConcatTransform(Collections.singletonList(BinaryString.fromString("****")))); setColumnMasking(identifier, masking); - for (String suffix : Arrays.asList("files", "file_key_ranges", "binlog")) { + for (String suffix : Arrays.asList("files", "file_key_ranges", "binlog", "statistics")) { Identifier sysId = Identifier.create( identifier.getDatabaseName(), @@ -5947,6 +6021,10 @@ void testQueryAuthRejectedWhereItCannotBeEnforced() throws Exception { assertThatThrownBy(() -> new LocalTableQuery((FileStoreTable) table)) .isInstanceOf(UnsupportedOperationException.class) .hasMessageContaining("query-auth table"); + + assertThatThrownBy(() -> ((FileStoreTable) table).newVectorSearchBuilder().newVectorRead()) + .isInstanceOf(UnsupportedOperationException.class) + .hasMessageContaining("query-auth table"); } @Test @@ -5980,6 +6058,105 @@ void testRowIdFilterOnDataEvolutionQueryAuthTable() throws Exception { assertThat(readBuilder.newScan().plan().splits()).isNotEmpty(); } + @Test + void testSuppliedGlobalIndexResultIgnoredUnderQueryAuth() throws Exception { + Identifier identifier = Identifier.create("test_table_db", "auth_de_supplied_index"); + List fields = new ArrayList<>(); + fields.add(new DataField(0, "f0", DataTypes.INT())); + Map options = new HashMap<>(); + options.put(CoreOptions.ROW_TRACKING_ENABLED.key(), "true"); + options.put(CoreOptions.DATA_EVOLUTION_ENABLED.key(), "true"); + options.put(CoreOptions.SOURCE_SPLIT_TARGET_SIZE.key(), "1 b"); + Table table = createMaskingAuthTable(identifier, fields, options); + + for (int v : new int[] {10, 20, 30}) { + BatchWriteBuilder builder = table.newBatchWriteBuilder(); + try (BatchTableWrite write = builder.newWrite()) { + write.write(GenericRow.of(v)); + builder.newCommit().commit(write.prepareCommit()); + } + } + + InnerTableScan scan = (InnerTableScan) table.newReadBuilder().newScan(); + scan.withGlobalIndexResult(GlobalIndexResult.fromRange(new Range(0, 1))); + assertThat(scan.plan().splits()).hasSize(3); + } + + @Test + void testRowIdFilterWithLimitOnDataEvolutionQueryAuthTable() throws Exception { + Identifier identifier = Identifier.create("test_table_db", "auth_de_rowid_limit"); + List fields = new ArrayList<>(); + fields.add(new DataField(0, "f0", DataTypes.INT())); + Map options = new HashMap<>(); + options.put(CoreOptions.ROW_TRACKING_ENABLED.key(), "true"); + options.put(CoreOptions.DATA_EVOLUTION_ENABLED.key(), "true"); + options.put(CoreOptions.SOURCE_SPLIT_TARGET_SIZE.key(), "1 b"); + Table table = createMaskingAuthTable(identifier, fields, options); + + for (int v : new int[] {10, 20, 30}) { + BatchWriteBuilder builder = table.newBatchWriteBuilder(); + try (BatchTableWrite write = builder.newWrite()) { + write.write(GenericRow.of(v)); + builder.newCommit().commit(write.prepareCommit()); + } + } + + Predicate onLastRowId = + LeafPredicate.of( + new FieldTransform( + new FieldRef( + SpecialFields.ROW_ID.id(), + SpecialFields.ROW_ID.name(), + DataTypes.BIGINT())), + Equal.INSTANCE, + Collections.singletonList(2L)); + ReadBuilder readBuilder = table.newReadBuilder().withFilter(onLastRowId).withLimit(1); + assertThat(readBuilder.newScan().plan().splits()).hasSize(3); + } + + @Test + void testMaskedRowIdFilterOnDataEvolutionQueryAuthTable() throws Exception { + Identifier identifier = Identifier.create("test_table_db", "auth_de_masked_rowid"); + List fields = new ArrayList<>(); + fields.add(new DataField(0, "f0", DataTypes.BIGINT())); + Map options = new HashMap<>(); + options.put(CoreOptions.ROW_TRACKING_ENABLED.key(), "true"); + options.put(CoreOptions.DATA_EVOLUTION_ENABLED.key(), "true"); + Table table = createMaskingAuthTable(identifier, fields, options); + + BatchWriteBuilder builder = table.newBatchWriteBuilder(); + try (BatchTableWrite write = builder.newWrite()) { + write.write(GenericRow.of(42L)); + builder.newCommit().commit(write.prepareCommit()); + } + + Map masking = new HashMap<>(); + masking.put( + SpecialFields.ROW_ID.name(), + new FieldTransform(new FieldRef(0, "f0", DataTypes.BIGINT()))); + setColumnMasking(identifier, masking); + + Predicate onMaskedRowId = + LeafPredicate.of( + new FieldTransform( + new FieldRef( + SpecialFields.ROW_ID.id(), + SpecialFields.ROW_ID.name(), + DataTypes.BIGINT())), + Equal.INSTANCE, + Collections.singletonList(42L)); + RowType readType = + new RowType(Arrays.asList(table.rowType().getField("f0"), SpecialFields.ROW_ID)); + ReadBuilder readBuilder = + table.newReadBuilder().withReadType(readType).withFilter(onMaskedRowId); + List rows = + collectRows( + readBuilder.newRead().createReader(readBuilder.newScan().plan().splits()), + readType); + assertThat(rows).hasSize(1); + assertThat(rows.get(0).getLong(1)).isEqualTo(42L); + } + @Test void testRowFilterWithTopNKeepsAuthorizedSplits() throws Exception { Identifier identifier = Identifier.create("test_table_db", "auth_table_topn"); diff --git a/paimon-core/src/test/java/org/apache/paimon/schema/SchemaValidationTest.java b/paimon-core/src/test/java/org/apache/paimon/schema/SchemaValidationTest.java index 5f5aa2fce0eb..b7da752f7b06 100644 --- a/paimon-core/src/test/java/org/apache/paimon/schema/SchemaValidationTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/schema/SchemaValidationTest.java @@ -769,6 +769,35 @@ mergeOptions, singletonList("id"), emptyList()))) "Primary-key managed BLOB tables do not support 'pk-clustering-override'."); } + @Test + public void testQueryAuthOnlyOnFileStoreTables() { + for (String type : Arrays.asList("format-table", "object-table")) { + Map options = new HashMap<>(); + options.put(CoreOptions.QUERY_AUTH_ENABLED.key(), "true"); + options.put(CoreOptions.TYPE.key(), type); + assertThatThrownBy(() -> validateTableSchema(queryAuthSchema(options))) + .hasMessageContaining(CoreOptions.QUERY_AUTH_ENABLED.key()); + } + + for (String type : Arrays.asList("table", "materialized-table")) { + Map options = new HashMap<>(); + options.put(CoreOptions.QUERY_AUTH_ENABLED.key(), "true"); + options.put(CoreOptions.TYPE.key(), type); + validateTableSchema(queryAuthSchema(options)); + } + } + + private TableSchema queryAuthSchema(Map options) { + return new TableSchema( + 1, + singletonList(new DataField(0, "id", DataTypes.INT())), + 10, + emptyList(), + emptyList(), + options, + ""); + } + private TableSchema primaryKeyBlobSchema( Map options, List primaryKeys, List partitionKeys) { return new TableSchema( diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/vectorsearch/FlinkVectorSearchBuilderImpl.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/vectorsearch/FlinkVectorSearchBuilderImpl.java index cad0f8683536..d4bf9748634f 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/vectorsearch/FlinkVectorSearchBuilderImpl.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/vectorsearch/FlinkVectorSearchBuilderImpl.java @@ -40,6 +40,7 @@ public FlinkVectorSearchBuilderImpl(InnerTable table, StreamExecutionEnvironment @Override public VectorRead newVectorRead() { + rejectUnderQueryAuth(); checkNotNull(vector, "vector must be set via withVector()"); if (isPrimaryKeyVectorSearch()) { return new FlinkPrimaryKeyVectorRead( diff --git a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/read/SparkVectorSearchBuilderImpl.java b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/read/SparkVectorSearchBuilderImpl.java index a1748b6513cf..3c8eedc3a580 100644 --- a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/read/SparkVectorSearchBuilderImpl.java +++ b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/read/SparkVectorSearchBuilderImpl.java @@ -38,6 +38,7 @@ public SparkVectorSearchBuilderImpl(InnerTable table) { @Override public VectorRead newVectorRead() { + rejectUnderQueryAuth(); if (isPrimaryKeyVectorSearch()) { return new SparkPrimaryKeyVectorRead( table, vectorColumn, vector, limit, options, filter); From e4879307e6ffb3ed53dc9c20c92a0ed3747783f1 Mon Sep 17 00:00:00 2001 From: Li Jiajia Date: Wed, 26 Aug 2026 04:50:51 -0400 Subject: [PATCH 7/7] [core] Simplify the query-auth changes --- .../paimon/predicate/PredicateVisitor.java | 1 - .../apache/paimon/catalog/CatalogUtils.java | 14 +- .../paimon/catalog/TableQueryAuthResult.java | 87 ++-- .../globalindex/DataEvolutionBatchScan.java | 2 - .../paimon/operation/MergeFileSplitRead.java | 3 +- .../paimon/schema/SchemaValidation.java | 29 +- .../table/source/AbstractBatchTableScan.java | 11 +- .../table/source/AbstractDataTableRead.java | 49 +-- .../table/source/AbstractDataTableScan.java | 40 +- .../source/BatchVectorSearchBuilderImpl.java | 13 +- .../source/FullTextSearchBuilderImpl.java | 13 +- .../table/source/HybridSearchBuilderImpl.java | 12 +- .../paimon/table/source/ReadBuilderImpl.java | 5 +- .../table/source/VectorSearchBuilderImpl.java | 7 +- .../apache/paimon/rest/RESTCatalogTest.java | 413 ++++++------------ 15 files changed, 249 insertions(+), 450 deletions(-) diff --git a/paimon-common/src/main/java/org/apache/paimon/predicate/PredicateVisitor.java b/paimon-common/src/main/java/org/apache/paimon/predicate/PredicateVisitor.java index 99a6692e9e43..b8ca94f57e7f 100644 --- a/paimon-common/src/main/java/org/apache/paimon/predicate/PredicateVisitor.java +++ b/paimon-common/src/main/java/org/apache/paimon/predicate/PredicateVisitor.java @@ -40,7 +40,6 @@ static Set collectFieldNames(@Nullable Predicate predicate) { return predicate.visit(new FieldNameCollector()); } - /** Collects the field names a transform's inputs reference. */ static Set collectTransformFieldNames(Transform transform) { Set fieldNames = new HashSet<>(); for (Object input : transform.inputs()) { diff --git a/paimon-core/src/main/java/org/apache/paimon/catalog/CatalogUtils.java b/paimon-core/src/main/java/org/apache/paimon/catalog/CatalogUtils.java index 9afa62468ea9..916f63350c7b 100644 --- a/paimon-core/src/main/java/org/apache/paimon/catalog/CatalogUtils.java +++ b/paimon-core/src/main/java/org/apache/paimon/catalog/CatalogUtils.java @@ -32,6 +32,7 @@ import org.apache.paimon.rest.exceptions.NotImplementedException; import org.apache.paimon.schema.Schema; import org.apache.paimon.schema.SchemaManager; +import org.apache.paimon.schema.SchemaValidation; import org.apache.paimon.schema.TableSchema; import org.apache.paimon.table.CatalogEnvironment; import org.apache.paimon.table.FileStoreTable; @@ -168,17 +169,8 @@ public static void validateCreateTable(Schema schema, boolean dataTokenEnabled) if (tableType.equals(TableType.FORMAT_TABLE)) { validateFormatTableOptions(options, dataTokenEnabled); } - // only a file-store table reads through the auth reader; anywhere else the rules would - // be accepted and then silently not applied - if (options.get(CoreOptions.QUERY_AUTH_ENABLED) - && tableType != TableType.TABLE - && tableType != TableType.MATERIALIZED_TABLE) { - throw new IllegalArgumentException( - String.format( - "%s is not supported on a %s: its read does not apply row filters or " - + "column masks.", - CoreOptions.QUERY_AUTH_ENABLED.key(), tableType)); - } + SchemaValidation.validateQueryAuthTableType( + tableType, options.get(CoreOptions.QUERY_AUTH_ENABLED)); for (DataField field : schema.fields()) { validateDefaultValue(field.type(), field.defaultValue()); } diff --git a/paimon-core/src/main/java/org/apache/paimon/catalog/TableQueryAuthResult.java b/paimon-core/src/main/java/org/apache/paimon/catalog/TableQueryAuthResult.java index 166a702db234..e3fd81442b82 100644 --- a/paimon-core/src/main/java/org/apache/paimon/catalog/TableQueryAuthResult.java +++ b/paimon-core/src/main/java/org/apache/paimon/catalog/TableQueryAuthResult.java @@ -29,7 +29,9 @@ import org.apache.paimon.predicate.PredicateVisitor; import org.apache.paimon.predicate.Transform; import org.apache.paimon.reader.RecordReader; +import org.apache.paimon.table.FileStoreTable; import org.apache.paimon.table.SpecialFields; +import org.apache.paimon.table.Table; import org.apache.paimon.table.source.DataFilePlan; import org.apache.paimon.table.source.QueryAuthSplit; import org.apache.paimon.table.source.Split; @@ -39,6 +41,7 @@ import org.apache.paimon.types.RowType; import org.apache.paimon.utils.InternalRowUtils; import org.apache.paimon.utils.JsonSerdeUtil; +import org.apache.paimon.utils.ListUtils; import org.apache.paimon.utils.StringUtils; import javax.annotation.Nullable; @@ -88,11 +91,24 @@ public Map columnMasking() { return columnMasking; } - /** Whether this result carries any effective row-filter or masking rule. */ public boolean hasRules() { return extractPredicate() != null || !extractColumnMasking().isEmpty(); } + /** + * Rejects a search on a query-auth table. Called from the methods that produce a scan or a + * read, not from the builder constructors: the builders are serializable, and deserialization + * would skip a constructor check. + */ + public static void rejectSearchUnderQueryAuth(@Nullable Table table) { + if (table instanceof FileStoreTable + && ((FileStoreTable) table).coreOptions().queryAuthEnabled()) { + throw new UnsupportedOperationException( + "Search is not supported on a query-auth table: the index ranks raw values, " + + "which a column mask invalidates."); + } + } + /** * Drops the conjuncts of {@code predicate} referencing any of {@code fields}; returns null when * nothing remains. Used to keep raw-statistics pushdown off masked columns. @@ -124,6 +140,8 @@ private static Predicate filterConjuncts( if (kept.isEmpty()) { return null; } + // and() folds an always-true/false leaf into a field-less constant, which would drop the + // field names the callers match against; a lone conjunct must come back untouched return kept.size() == 1 ? kept.get(0) : PredicateBuilder.and(kept); } @@ -154,22 +172,17 @@ private static Set postMaskFilterFields( */ public Set authFields(List readFields, @Nullable Predicate filter) { Set postMask = postMaskFilterFields(filter, extractColumnMasking().keySet()); - List visible = readFields; - if (!postMask.isEmpty()) { - visible = new ArrayList<>(readFields); - for (String field : postMask) { - if (!visible.contains(field)) { - visible.add(field); - } - } - } + // requiredAuthFields de-duplicates, so a plain concatenation is enough here + List visible = + postMask.isEmpty() + ? readFields + : ListUtils.union(readFields, new ArrayList<>(postMask)); Set ruleFields = requiredAuthFields(visible); // requiredAuthFields returns what the rules read, not the operands themselves ruleFields.addAll(postMask); return ruleFields; } - /** Appends the missing {@code ruleFields} of {@code tableType} to {@code readType}. */ @Nullable public static RowType appendMissingFields( RowType tableType, RowType readType, Set ruleFields) { @@ -282,14 +295,15 @@ public void validateAgainstSchema(RowType tableType, @Nullable List proj // consumed unmasked and its raw value published through this target. Masking // the target of another mask is only self-consistent if composed, which the // read does not do; refuse the pair rather than leak. - if (!input.equals(target) && masking.containsKey(input)) { - throw new IllegalArgumentException( - String.format( - "Column masking on '%s' reads column '%s', which is masked " - + "too. The mask would be computed from the raw value " - + "of '%s' and expose it through '%s'.", - target, input, input, target)); - } + checkArgument( + input.equals(target) || !masking.containsKey(input), + "Column masking on '%s' reads column '%s', which is masked " + + "too. The mask would be computed from the raw value " + + "of '%s' and expose it through '%s'.", + target, + input, + input, + target); } } for (String operand : PredicateVisitor.collectFieldNames(extractPredicate())) { @@ -327,17 +341,18 @@ private static void checkNotRenamed( if (readType.containsField(field)) { // a dropped and re-added column keeps the name but gets a fresh id, so the same // name may be an unrelated column in the snapshot being read - if (readType.getField(field).id() != id) { - throw new IllegalArgumentException( - String.format( - "%s references column '%s' which the snapshot being read exposes " - + "as a different column of the same name (dropped and " - + "re-added since); refusing to read to avoid applying the " - + "rule to unrelated data.", - rule, field)); - } + checkArgument( + readType.getField(field).id() == id, + "%s references column '%s' which the snapshot being read exposes " + + "as a different column of the same name (dropped and " + + "re-added since); refusing to read to avoid applying the " + + "rule to unrelated data.", + rule, + field); return; } + // not checkArgument: its arguments are evaluated eagerly, and getField(id) throws when + // the id is absent, which is the normal case here if (readType.containsField(id)) { throw new IllegalArgumentException( String.format( @@ -390,14 +405,14 @@ private static void checkFieldExists( "%s references system column '%s' which the query does not project.", rule, field)); } - if (!tableType.containsField(field)) { - throw new IllegalArgumentException( - String.format( - "%s references column '%s' which does not exist in table schema %s. " - + "The rule may be stale after a column rename or drop; " - + "refusing to read.", - rule, field, tableType.getFieldNames())); - } + checkArgument( + tableType.containsField(field), + "%s references column '%s' which does not exist in table schema %s. " + + "The rule may be stale after a column rename or drop; " + + "refusing to read.", + rule, + field, + tableType.getFieldNames()); } /** diff --git a/paimon-core/src/main/java/org/apache/paimon/globalindex/DataEvolutionBatchScan.java b/paimon-core/src/main/java/org/apache/paimon/globalindex/DataEvolutionBatchScan.java index 589a9980e294..7a16a2e7accb 100644 --- a/paimon-core/src/main/java/org/apache/paimon/globalindex/DataEvolutionBatchScan.java +++ b/paimon-core/src/main/java/org/apache/paimon/globalindex/DataEvolutionBatchScan.java @@ -106,8 +106,6 @@ public InnerTableScan withFilter(Predicate predicate) { // the wrapped scan defers the filter but strips only masked columns; row ids must // go here, since data-evolution statistics carry logical columns only Predicate residual = rowIdSafeResidualFilter(predicate); - // what is left out reaches the reader only, so the wrapped scan never learns a - // filter exists and would let limit/TopN prune ahead of it rowIdFilterDeferred = containsRowId(predicate); if (residual != null) { batchScan.withFilter(residual); diff --git a/paimon-core/src/main/java/org/apache/paimon/operation/MergeFileSplitRead.java b/paimon-core/src/main/java/org/apache/paimon/operation/MergeFileSplitRead.java index 82eeaffdf6d1..9e66aad52fca 100644 --- a/paimon-core/src/main/java/org/apache/paimon/operation/MergeFileSplitRead.java +++ b/paimon-core/src/main/java/org/apache/paimon/operation/MergeFileSplitRead.java @@ -146,8 +146,7 @@ public MergeFileSplitRead withReadType(RowType readType) { readerFactoryBuilder.withReadValueType(adjustedReadType); mergeSorter.setProjectedValueType(adjustedReadType); - // Project away fields added for merging; reset any previous projection, as this - // method may be called again. + // reset rather than latch: this method may be called again outerReadType = adjustedReadType != readType ? readType : null; return this; diff --git a/paimon-core/src/main/java/org/apache/paimon/schema/SchemaValidation.java b/paimon-core/src/main/java/org/apache/paimon/schema/SchemaValidation.java index 0139b059d9d9..896612d30599 100644 --- a/paimon-core/src/main/java/org/apache/paimon/schema/SchemaValidation.java +++ b/paimon-core/src/main/java/org/apache/paimon/schema/SchemaValidation.java @@ -156,18 +156,9 @@ public static void validateTableSchema(TableSchema schema, Set dynamicOp validateOnlyContainPrimitiveType(schema.fields(), schema.primaryKeys(), "primary key"); validateOnlyContainPrimitiveType(schema.fields(), schema.partitionKeys(), "partition"); - // only a file-store table reads through the auth reader; reject here rather than only at - // create time, so ALTER cannot turn the option on for a table type that ignores it - TableType tableType = options.type(); - if (options.queryAuthEnabled() - && tableType != TableType.TABLE - && tableType != TableType.MATERIALIZED_TABLE) { - throw new RuntimeException( - String.format( - "%s is not supported on a %s: its read does not apply row filters or " - + "column masks.", - CoreOptions.QUERY_AUTH_ENABLED.key(), tableType)); - } + // reject here rather than only at create time, so ALTER cannot turn the option on for a + // table type that ignores it + validateQueryAuthTableType(options.type(), options.queryAuthEnabled()); if (options.primaryKeyNullable() && schema.primaryKeys().isEmpty()) { throw new IllegalArgumentException( @@ -412,6 +403,20 @@ public static void validateTableSchema(TableSchema schema, Set dynamicOp validateManifestSort(schema, options); } + /** + * Only a file-store table reads through the auth reader; anywhere else the rules would be + * accepted and then silently not applied. + */ + public static void validateQueryAuthTableType(TableType tableType, boolean queryAuthEnabled) { + checkArgument( + !queryAuthEnabled + || tableType == TableType.TABLE + || tableType == TableType.MATERIALIZED_TABLE, + "%s is not supported on a %s: its read does not apply row filters or column masks.", + CoreOptions.QUERY_AUTH_ENABLED.key(), + tableType); + } + public static void validateFallbackBranch(SchemaManager schemaManager, TableSchema schema) { String fallbackBranch = schema.options().get(CoreOptions.SCAN_FALLBACK_BRANCH.key()); String primaryBranch = schema.options().get(CoreOptions.SCAN_PRIMARY_BRANCH.key()); diff --git a/paimon-core/src/main/java/org/apache/paimon/table/source/AbstractBatchTableScan.java b/paimon-core/src/main/java/org/apache/paimon/table/source/AbstractBatchTableScan.java index 8a3cd61ee979..1abcfcea306d 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/source/AbstractBatchTableScan.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/source/AbstractBatchTableScan.java @@ -19,7 +19,6 @@ package org.apache.paimon.table.source; import org.apache.paimon.CoreOptions; -import org.apache.paimon.catalog.TableQueryAuthResult; import org.apache.paimon.manifest.PartitionEntry; import org.apache.paimon.predicate.Predicate; import org.apache.paimon.predicate.SortValue; @@ -42,7 +41,6 @@ import java.time.Duration; import java.util.ArrayList; -import java.util.Collections; import java.util.List; import java.util.Optional; import java.util.OptionalLong; @@ -156,14 +154,7 @@ public List listPartitionEntries() { // partition listing bypasses plan(), so apply the rules here too: without the row filter // it would report partitions the caller cannot read, and pushing a filter on a masked // column against raw partition values would drop partitions the query matches - TableQueryAuthResult authResult = authQuery(); - applyAuthFilter(authResult == null ? null : authResult.extractPredicate()); - this.authMaskedFields = - authResult == null - ? Collections.emptySet() - : authResult.extractColumnMasking().keySet(); - rejectMaskedPartitionFilter(); - ensureFilterPushdown(authMaskedFields); + applyAuthRules(); if (startingScanner == null) { startingScanner = createStartingScanner(false); } diff --git a/paimon-core/src/main/java/org/apache/paimon/table/source/AbstractDataTableRead.java b/paimon-core/src/main/java/org/apache/paimon/table/source/AbstractDataTableRead.java index 38474d0fa517..7b14dd1cac69 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/source/AbstractDataTableRead.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/source/AbstractDataTableRead.java @@ -55,8 +55,7 @@ public abstract class AbstractDataTableRead implements InnerTableRead { // as read-level TopN already does (see ReadBuilderImpl) private final boolean queryAuthEnabled; - // the read type the subclass reads with; widened for auth when needed, and fixed - // once a reader exists (split reads cache their format readers) + // the auth-widened read type currently applied, or null when the plain read type is @Nullable private RowType appliedReadType; // blob-view columns that only resolve through the dedicated blob-view read path @@ -114,7 +113,7 @@ public final InnerTableRead withProjection(int[] projection) { @Override public final InnerTableRead withReadType(RowType readType) { this.readType = readType; - this.appliedReadType = readType; + this.appliedReadType = null; applyReadType(readType); return this; } @@ -148,15 +147,17 @@ protected final QueryAuthContext unwrapQueryAuthSplit(Split split) { protected final RecordReader createDataReader( Split split, @Nullable TableQueryAuthResult authResult) throws IOException { - // a TableRead is reused across splits; auth may have widened the projection for the - // previous one, so restore the requested type before deciding this split's widening + // A TableRead can be reused for multiple splits. Authentication may have expanded an + // explicitly configured physical projection for the previous split, so restore it before + // applying the current split's authorization dependencies. Without an explicit projection, + // the underlying reader must retain its own default read type. if (readType != null) { applyReadType(readType); appliedReadType = null; } RecordReader reader; if (authResult == null) { - reader = backProject(readSplit(split)); + reader = backProject(reader(split)); } else { reader = authedReader(split, authResult); } @@ -167,10 +168,6 @@ protected final RecordReader createDataReader( return reader; } - private RecordReader readSplit(Split split) throws IOException { - return reader(split); - } - private RecordReader authedReader(Split split, TableQueryAuthResult authResult) throws IOException { List readFields = currentReadType().getFieldNames(); @@ -185,45 +182,30 @@ private RecordReader authedReader(Split split, TableQueryAuthResult } // the split read emits appliedReadType; rules are remapped against it by name RowType outputType = appliedReadType != null ? appliedReadType : currentReadType(); - if (widened != null && !widened.equals(outputType)) { - // rules changed after the read schema was fixed: fail clearly if they no longer fit - List outputFields = outputType.getFieldNames(); - for (String field : widened.getFieldNames()) { - if (!outputFields.contains(field)) { - throw new IllegalStateException( - String.format( - "Query auth rules changed and now require column '%s', but the " - + "read schema is already fixed to %s. Recreate the " - + "reader to apply the new rules.", - field, outputFields)); - } - } - } // masks apply only to columns readable from the query: the ones it projects plus the // ones the rules pulled in; a mask on anything else is inert Map masking = authResult.extractColumnMasking(); - Map selectedMasking = Collections.emptyMap(); + Map selectedColumnMasking = Collections.emptyMap(); if (!masking.isEmpty()) { Set activeFields = new HashSet<>(readFields); activeFields.addAll(ruleFields); - selectedMasking = new HashMap<>(); + selectedColumnMasking = new HashMap<>(); for (Map.Entry mask : masking.entrySet()) { if (activeFields.contains(mask.getKey())) { - selectedMasking.put(mask.getKey(), mask.getValue()); + selectedColumnMasking.put(mask.getKey(), mask.getValue()); } } } RecordReader reader = authResult.doAuth( - readSplit(split), + reader(split), outputType, authResult.extractPredicate(), - selectedMasking); + selectedColumnMasking); reader = filterMaskedConjuncts(reader, outputType, maskedFilterFields); return backProject(reader); } - /** The columns of the query filter that the current auth rules mask. */ private Set maskedFilterFields(Set maskTargets) { if (predicate == null || maskTargets.isEmpty()) { return Collections.emptySet(); @@ -261,12 +243,9 @@ private RecordReader filterMaskedConjuncts( return reader.filter(filter::test); } - /** - * Project auth-widened rows back to the query's read type — on every split, since the widened - * read schema stays in effect even for splits without auth rules. - */ + /** Project auth-widened rows back to the read type the query asked for. */ private RecordReader backProject(RecordReader reader) { - if (appliedReadType == null || appliedReadType == readType) { + if (appliedReadType == null) { return reader; } ProjectedRow backRow = diff --git a/paimon-core/src/main/java/org/apache/paimon/table/source/AbstractDataTableScan.java b/paimon-core/src/main/java/org/apache/paimon/table/source/AbstractDataTableScan.java index 95507f411975..24bf32e50cf0 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/source/AbstractDataTableScan.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/source/AbstractDataTableScan.java @@ -124,8 +124,12 @@ protected AbstractDataTableScan( this.queryAuth = queryAuth; } - @Override - public final TableScan.Plan plan() { + /** + * Refreshes the auth state and pushes what can be pushed. Every path that reaches the files + * must run this, so it stays the one place the order of these steps is defined. + */ + @Nullable + protected final TableQueryAuthResult applyAuthRules() { TableQueryAuthResult queryAuthResult = authQuery(); // Always apply/clear the auth filter so removing auth leaves no stale partition pruning. applyAuthFilter(queryAuthResult == null ? null : queryAuthResult.extractPredicate()); @@ -134,7 +138,13 @@ public final TableScan.Plan plan() { ? Collections.emptySet() : queryAuthResult.extractColumnMasking().keySet(); rejectMaskedPartitionFilter(); - ensureFilterPushdown(authMaskedFields); + ensureFilterPushdown(); + return queryAuthResult; + } + + @Override + public final TableScan.Plan plan() { + TableQueryAuthResult queryAuthResult = applyAuthRules(); applyAuthReadType(queryAuthResult); Plan plan = planWithoutAuth(); if (queryAuthResult != null) { @@ -145,7 +155,7 @@ public final TableScan.Plan plan() { protected abstract TableScan.Plan planWithoutAuth(); - protected void applyAuthFilter(@Nullable Predicate authPredicate) { + private void applyAuthFilter(@Nullable Predicate authPredicate) { if (Objects.equals(authPredicate, appliedAuthPredicate)) { return; } @@ -323,7 +333,7 @@ private Set partitionPredicateFields(PartitionPredicate partitionPredica * evaluation, so it can never be re-checked on the masked value. Fail closed. One routed * through withFilter is fine: that path defers it and evaluates it post-mask. */ - protected void rejectMaskedPartitionFilter() { + private void rejectMaskedPartitionFilter() { if (partitionFilterFields.isEmpty() || authMaskedFields.isEmpty()) { return; } @@ -344,11 +354,11 @@ protected void rejectMaskedPartitionFilter() { * Pushes the query filter once, minus the conjuncts on masked columns. Also called by partition * listing; a mask found later on an already-pushed column fails closed. */ - protected final void ensureFilterPushdown(Set maskedFields) { + private void ensureFilterPushdown() { if (userFilter == null) { return; } - Set maskedInFilter = new HashSet<>(maskedFields); + Set maskedInFilter = new HashSet<>(authMaskedFields); maskedInFilter.retainAll(PredicateVisitor.collectFieldNames(userFilter)); if (!maskedInFilter.isEmpty()) { // masked conjuncts drop rows at read time only: keep limit/TopN pruning off @@ -393,15 +403,13 @@ private void applyAuthReadType(@Nullable TableQueryAuthResult queryAuthResult) { } } // never narrow within this scan's lifetime: readers fix their schema on first use - if (appliedScanReadType != null) { - RowType widened = - TableQueryAuthResult.appendMissingFields( - appliedScanReadType, - desired, - new HashSet<>(appliedScanReadType.getFieldNames())); - if (widened != null) { - desired = widened; - } + RowType widenedToApplied = + TableQueryAuthResult.appendMissingFields( + appliedScanReadType, + desired, + new HashSet<>(appliedScanReadType.getFieldNames())); + if (widenedToApplied != null) { + desired = widenedToApplied; } if (!desired.equals(appliedScanReadType)) { snapshotReader.withReadType(desired); diff --git a/paimon-core/src/main/java/org/apache/paimon/table/source/BatchVectorSearchBuilderImpl.java b/paimon-core/src/main/java/org/apache/paimon/table/source/BatchVectorSearchBuilderImpl.java index f9b83eeab470..fd02d0011c40 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/source/BatchVectorSearchBuilderImpl.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/source/BatchVectorSearchBuilderImpl.java @@ -18,6 +18,7 @@ package org.apache.paimon.table.source; +import org.apache.paimon.catalog.TableQueryAuthResult; import org.apache.paimon.partition.PartitionPredicate; import org.apache.paimon.predicate.Predicate; import org.apache.paimon.predicate.PredicateBuilder; @@ -125,7 +126,7 @@ public BatchVectorSearchBuilder withOption(String key, String value) { @Override public VectorScan newVectorScan() { - rejectUnderQueryAuth(); + TableQueryAuthResult.rejectSearchUnderQueryAuth(table); if (isPrimaryKeyVectorSearch()) { return new PrimaryKeyVectorScan( table, @@ -139,7 +140,7 @@ public VectorScan newVectorScan() { @Override public BatchVectorRead newBatchVectorRead() { - rejectUnderQueryAuth(); + TableQueryAuthResult.rejectSearchUnderQueryAuth(table); checkArgument(limit > 0, "Limit must be positive, set via withLimit()"); checkNotNull(vectorColumn, "Vector column must be set via withVectorColumn()"); checkArgument( @@ -159,12 +160,4 @@ protected boolean isPrimaryKeyVectorSearch() { return vectorColumn != null && table.coreOptions().primaryKeyVectorIndexColumns().contains(vectorColumn.name()); } - - private void rejectUnderQueryAuth() { - if (table.coreOptions().queryAuthEnabled()) { - throw new UnsupportedOperationException( - "Search is not supported on a query-auth table: the index ranks raw values, " - + "which a column mask invalidates."); - } - } } diff --git a/paimon-core/src/main/java/org/apache/paimon/table/source/FullTextSearchBuilderImpl.java b/paimon-core/src/main/java/org/apache/paimon/table/source/FullTextSearchBuilderImpl.java index 27bcaa5a9270..77d74c9a5f6c 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/source/FullTextSearchBuilderImpl.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/source/FullTextSearchBuilderImpl.java @@ -19,6 +19,7 @@ package org.apache.paimon.table.source; import org.apache.paimon.Snapshot; +import org.apache.paimon.catalog.TableQueryAuthResult; import org.apache.paimon.index.pk.PrimaryKeyIndexDefinition; import org.apache.paimon.index.pk.PrimaryKeyIndexDefinitions; import org.apache.paimon.partition.PartitionPredicate; @@ -72,7 +73,7 @@ public FullTextSearchBuilder withQuery(String fieldName, String query) { @Override public FullTextScan newFullTextScan() { - rejectUnderQueryAuth(); + TableQueryAuthResult.rejectSearchUnderQueryAuth(table); DataField textColumn = textColumn(); Optional definition = primaryKeyFullTextDefinition(textColumn); return definition.isPresent() @@ -87,7 +88,7 @@ public FullTextScan newFullTextScan() { @Override public FullTextRead newFullTextRead() { - rejectUnderQueryAuth(); + TableQueryAuthResult.rejectSearchUnderQueryAuth(table); checkArgument(limit > 0, "Limit must be positive, set via withLimit()"); DataField textColumn = textColumn(); Optional definition = primaryKeyFullTextDefinition(textColumn); @@ -130,12 +131,4 @@ FullTextSearchBuilderImpl withSnapshot(Snapshot snapshot) { this.pinnedSnapshot = snapshot; return this; } - - private void rejectUnderQueryAuth() { - if (table.coreOptions().queryAuthEnabled()) { - throw new UnsupportedOperationException( - "Search is not supported on a query-auth table: the index ranks raw values, " - + "which a column mask invalidates."); - } - } } diff --git a/paimon-core/src/main/java/org/apache/paimon/table/source/HybridSearchBuilderImpl.java b/paimon-core/src/main/java/org/apache/paimon/table/source/HybridSearchBuilderImpl.java index ac9b965c2e5b..35460de7b472 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/source/HybridSearchBuilderImpl.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/source/HybridSearchBuilderImpl.java @@ -19,6 +19,7 @@ package org.apache.paimon.table.source; import org.apache.paimon.Snapshot; +import org.apache.paimon.catalog.TableQueryAuthResult; import org.apache.paimon.data.BinaryRow; import org.apache.paimon.globalindex.GlobalIndexResult; import org.apache.paimon.globalindex.HybridSearchRanker; @@ -128,7 +129,7 @@ public HybridSearchBuilder withWeightedScoreRanker() { @Override public List routeBuilders() { - rejectUnderQueryAuth(); + TableQueryAuthResult.rejectSearchUnderQueryAuth(table); validateSearch(); Snapshot snapshot = null; @@ -378,13 +379,4 @@ protected FullTextSearchBuilder newFullTextSearchBuilder(HybridSearchRoute route } return fullTextSearchBuilder; } - - private void rejectUnderQueryAuth() { - if (table instanceof FileStoreTable - && ((FileStoreTable) table).coreOptions().queryAuthEnabled()) { - throw new UnsupportedOperationException( - "Search is not supported on a query-auth table: the index ranks raw values, " - + "which a column mask invalidates."); - } - } } diff --git a/paimon-core/src/main/java/org/apache/paimon/table/source/ReadBuilderImpl.java b/paimon-core/src/main/java/org/apache/paimon/table/source/ReadBuilderImpl.java index ff088731ff6c..ee6949eddb0c 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/source/ReadBuilderImpl.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/source/ReadBuilderImpl.java @@ -245,9 +245,8 @@ public TableRead newRead() { read.withReadType(readType); } if (queryAuthEnabled) { - // Skip TopN and, with a filter or a TopN present, the limit as well: the engine - // re-applies them. The reader does not evaluate the query filter on an auth-enabled - // table, so capping the rows here would cut away the rows that actually match. + // the reader does not evaluate the query filter on an auth-enabled table, so + // capping the rows here would cut away rows that actually match if (topN == null && limit != null) { return new LimitTableRead(read, limit, filter != null); } diff --git a/paimon-core/src/main/java/org/apache/paimon/table/source/VectorSearchBuilderImpl.java b/paimon-core/src/main/java/org/apache/paimon/table/source/VectorSearchBuilderImpl.java index 67839191a365..b5e572b8752b 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/source/VectorSearchBuilderImpl.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/source/VectorSearchBuilderImpl.java @@ -19,6 +19,7 @@ package org.apache.paimon.table.source; import org.apache.paimon.Snapshot; +import org.apache.paimon.catalog.TableQueryAuthResult; import org.apache.paimon.partition.PartitionPredicate; import org.apache.paimon.predicate.Predicate; import org.apache.paimon.predicate.PredicateBuilder; @@ -164,10 +165,6 @@ public VectorSearchBuilderImpl withSnapshot(Snapshot snapshot) { } protected void rejectUnderQueryAuth() { - if (table.coreOptions().queryAuthEnabled()) { - throw new UnsupportedOperationException( - "Search is not supported on a query-auth table: the index ranks raw values, " - + "which a column mask invalidates."); - } + TableQueryAuthResult.rejectSearchUnderQueryAuth(table); } } diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTest.java b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTest.java index 24ed7e285327..6d68f64b9b42 100644 --- a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTest.java @@ -60,6 +60,7 @@ import org.apache.paimon.predicate.FieldTransform; import org.apache.paimon.predicate.GreaterOrEqual; import org.apache.paimon.predicate.GreaterThan; +import org.apache.paimon.predicate.LeafFunction; import org.apache.paimon.predicate.LeafPredicate; import org.apache.paimon.predicate.Predicate; import org.apache.paimon.predicate.PredicateBuilder; @@ -103,6 +104,7 @@ import org.apache.paimon.table.source.TableScan; import org.apache.paimon.table.system.SystemTableLoader; import org.apache.paimon.types.DataField; +import org.apache.paimon.types.DataType; import org.apache.paimon.types.DataTypes; import org.apache.paimon.types.RowType; import org.apache.paimon.utils.InternalRowUtils; @@ -3931,6 +3933,28 @@ private Table createMaskingAuthTable( return catalog.getTable(identifier); } + private static LeafPredicate leaf( + int index, String name, DataType type, LeafFunction function, Object literal) { + return LeafPredicate.of( + new FieldTransform(new FieldRef(index, name, type)), + function, + Collections.singletonList(literal)); + } + + private void setColumnMask(Identifier identifier, String target, Transform transform) { + Map masking = new HashMap<>(); + masking.put(target, transform); + setColumnMasking(identifier, masking); + } + + /** Masks {@code target} with the constant {@code ****}. */ + private void maskConstant(Identifier identifier, String target) { + setColumnMask( + identifier, + target, + new ConcatTransform(Collections.singletonList(BinaryString.fromString("****")))); + } + private static List stringFields(String... names) { List fields = new ArrayList<>(); for (int i = 0; i < names.length; i++) { @@ -3960,15 +3984,14 @@ private static void writeStringRow(Table table, String... values) throws Excepti } private void maskDisplayWithFullName(Identifier identifier) { - Map columnMasking = new HashMap<>(); - columnMasking.put( + setColumnMask( + identifier, "display", new ConcatWsTransform( Arrays.asList( BinaryString.fromString("-"), new FieldRef(0, "first", DataTypes.STRING()), new FieldRef(1, "last", DataTypes.STRING())))); - setColumnMasking(identifier, columnMasking); } private static List collectRows(RecordReader reader, RowType rowType) @@ -4053,10 +4076,12 @@ void testColumnMaskingOnRowFilterColumnWithProjection() throws Exception { writeStringRow(table, "john", "doe", "secret", "o1"); LeafPredicate displayFilter = - LeafPredicate.of( - new FieldTransform(new FieldRef(2, "display", DataTypes.STRING())), + leaf( + 2, + "display", + DataTypes.STRING(), Equal.INSTANCE, - Collections.singletonList(BinaryString.fromString("secret"))); + BinaryString.fromString("secret")); setRowFilter(identifier, Collections.singletonList(displayFilter)); maskDisplayWithFullName(identifier); @@ -4175,14 +4200,13 @@ void testColumnMaskingPreservesNestedProjection() throws Exception { assertThat(rows).hasSize(1); assertThat(rows.get(0).getRow(1, 1).getString(0).toString()).isEqualTo("BV"); - Map columnMasking = new HashMap<>(); - columnMasking.put( + setColumnMask( + identifier, "display", new ConcatWsTransform( Arrays.asList( BinaryString.fromString("-"), new FieldRef(4, "extra", DataTypes.STRING())))); - setColumnMasking(identifier, columnMasking); readBuilder = table.newReadBuilder().withReadType(prunedReadType); rows = @@ -4205,23 +4229,18 @@ void testColumnMaskingStaleRuleFailsClosed() throws Exception { Collections.emptyMap()); writeStringRow(table, "john", "doe", "secret"); - Map staleTarget = new HashMap<>(); - staleTarget.put( - "renamed_away", - new ConcatTransform(Collections.singletonList(BinaryString.fromString("****")))); - setColumnMasking(identifier, staleTarget); + maskConstant(identifier, "renamed_away"); assertThatThrownBy(() -> readFully(table)) .isInstanceOf(IllegalArgumentException.class) .hasMessageContaining("does not exist in table schema"); - Map staleInput = new HashMap<>(); - staleInput.put( + setColumnMask( + identifier, "display", new ConcatWsTransform( Arrays.asList( BinaryString.fromString("-"), new FieldRef(0, "ghost", DataTypes.STRING())))); - setColumnMasking(identifier, staleInput); assertThatThrownBy(() -> readFully(table)) .isInstanceOf(IllegalArgumentException.class) .hasMessageContaining("does not exist in table schema"); @@ -4300,11 +4319,7 @@ void testColumnMaskingRejectsNestedPrunedMaskTarget() throws Exception { RowType tableRowType = table.rowType(); DataField sField = tableRowType.getField("s"); - Map masking = new HashMap<>(); - masking.put( - "s", - new ConcatTransform(Collections.singletonList(BinaryString.fromString("****")))); - setColumnMasking(identifier, masking); + maskConstant(identifier, "s"); RowType prunedS = ((RowType) sField.type()).project("b"); RowType prunedReadType = @@ -4338,11 +4353,7 @@ void testColumnMaskingOnColumnAddedAfterSnapshot() throws Exception { identifier, Collections.singletonList(SchemaChange.addColumn("extra", DataTypes.STRING())), false); - Map masking = new HashMap<>(); - masking.put( - "extra", - new ConcatTransform(Collections.singletonList(BinaryString.fromString("****")))); - setColumnMasking(identifier, masking); + maskConstant(identifier, "extra"); Table latest = catalog.getTable(identifier); ReadBuilder latestRead = latest.newReadBuilder(); @@ -4378,11 +4389,7 @@ void testColumnMaskingRenamedColumnTimeTravelFailsClosed() throws Exception { identifier, Collections.singletonList(SchemaChange.renameColumn("secret", "masked_secret")), false); - Map masking = new HashMap<>(); - masking.put( - "masked_secret", - new ConcatTransform(Collections.singletonList(BinaryString.fromString("****")))); - setColumnMasking(identifier, masking); + maskConstant(identifier, "masked_secret"); Table latest = catalog.getTable(identifier); ReadBuilder latestRead = latest.newReadBuilder(); @@ -4412,11 +4419,7 @@ void testColumnMaskingSystemTargetInertWhenUnprojected() throws Exception { Collections.singletonMap(CoreOptions.ROW_TRACKING_ENABLED.key(), "true")); writeStringRow(table, "d1", "o1"); - Map masking = new HashMap<>(); - masking.put( - "_ROW_ID", - new ConcatTransform(Collections.singletonList(BinaryString.fromString("****")))); - setColumnMasking(identifier, masking); + maskConstant(identifier, "_ROW_ID"); ReadBuilder readBuilder = table.newReadBuilder().withProjection(new int[] {0}); List rows = @@ -4444,11 +4447,10 @@ void testColumnMaskingInertTargetWithRenamedInputTimeTravel() throws Exception { identifier, Collections.singletonList(SchemaChange.addColumn("display", DataTypes.STRING())), false); - Map masking = new HashMap<>(); - masking.put( + setColumnMask( + identifier, "display", new FieldTransform(new FieldRef(1, "renamed_input", DataTypes.STRING()))); - setColumnMasking(identifier, masking); Table old = catalog.getTable(identifier) @@ -4502,11 +4504,7 @@ void testColumnMaskingRenamedUnderLiveScanFailsClosed() throws Exception { identifier, stringFields("first", "secret"), Collections.emptyMap()); writeStringRow(table, "john", "s1"); - Map masking = new HashMap<>(); - masking.put( - "secret", - new ConcatTransform(Collections.singletonList(BinaryString.fromString("****")))); - setColumnMasking(identifier, masking); + maskConstant(identifier, "secret"); StreamTableScan scan = table.newReadBuilder().newStreamScan(); scan.plan(); @@ -4549,11 +4547,10 @@ void testColumnMaskingRejectsNestedPrunedRuleInput() throws Exception { RowType tableRowType = table.rowType(); DataField sField = tableRowType.getField("s"); - Map masking = new HashMap<>(); - masking.put( + setColumnMask( + identifier, "display", new CastTransform(new FieldRef(1, "s", sField.type()), DataTypes.STRING())); - setColumnMasking(identifier, masking); RowType prunedS = ((RowType) sField.type()).project("b"); RowType prunedReadType = @@ -4614,14 +4611,13 @@ void testColumnMaskingWithDataEvolutionColumnFiles() throws Exception { builder.newCommit().commit(commitables); } - Map masking = new HashMap<>(); - masking.put( + setColumnMask( + identifier, "f1", new ConcatWsTransform( Arrays.asList( BinaryString.fromString("-"), new FieldRef(2, "f2", DataTypes.STRING())))); - setColumnMasking(identifier, masking); ReadBuilder readBuilder = table.newReadBuilder().withProjection(new int[] {1}); List rows = @@ -4661,14 +4657,13 @@ void testColumnMaskingRejectsUnprojectedBlobViewInput() throws Exception { write.close(); commit.close(); - Map masking = new HashMap<>(); - masking.put( + setColumnMask( + identifier, "display", new ConcatWsTransform( Arrays.asList( BinaryString.fromString("-"), new FieldRef(1, "image", DataTypes.STRING())))); - setColumnMasking(identifier, masking); ReadBuilder readBuilder = table.newReadBuilder().withProjection(new int[] {0}); assertThatThrownBy( @@ -4696,18 +4691,13 @@ void testColumnMaskingDisablesTopNPushdown() throws Exception { Collections.singletonMap( CoreOptions.SOURCE_SPLIT_TARGET_SIZE.key(), "1 b")); for (int[] row : new int[][] {{100, 0}, {50, 1000}}) { - BatchWriteBuilder writeBuilder = table.newBatchWriteBuilder(); - BatchTableWrite write = writeBuilder.newWrite(); - write.write(GenericRow.of(row[0], row[1])); - BatchTableCommit commit = writeBuilder.newCommit(); - commit.commit(write.prepareCommit()); - write.close(); - commit.close(); + commitRows(table, GenericRow.of(row[0], row[1])); } - Map masking = new HashMap<>(); - masking.put("display", new FieldTransform(new FieldRef(1, "source", DataTypes.INT()))); - setColumnMasking(identifier, masking); + setColumnMask( + identifier, + "display", + new FieldTransform(new FieldRef(1, "source", DataTypes.INT()))); TopN topN = new TopN(new FieldRef(0, "display", DataTypes.INT()), DESCENDING, NULLS_LAST, 1); @@ -4736,19 +4726,12 @@ void testColumnMaskingReadingSystemField() throws Exception { fields, Collections.singletonMap(CoreOptions.ROW_TRACKING_ENABLED.key(), "true")); - BatchWriteBuilder writeBuilder = table.newBatchWriteBuilder(); - BatchTableWrite write = writeBuilder.newWrite(); - write.write(GenericRow.of(42L)); - BatchTableCommit commit = writeBuilder.newCommit(); - commit.commit(write.prepareCommit()); - write.close(); - commit.close(); + commitRows(table, GenericRow.of(42L)); - Map masking = new HashMap<>(); - masking.put( + setColumnMask( + identifier, "display", new FieldTransform(new FieldRef(0, "_ROW_ID", DataTypes.BIGINT().notNull()))); - setColumnMasking(identifier, masking); RowType readType = new RowType( @@ -4809,23 +4792,12 @@ void testColumnMaskingDisablesFilterStatsPruning() throws Exception { Collections.singletonMap( CoreOptions.SOURCE_SPLIT_TARGET_SIZE.key(), "1 b")); for (int[] row : new int[][] {{1, 1000}, {900, 10}}) { - BatchWriteBuilder writeBuilder = table.newBatchWriteBuilder(); - BatchTableWrite write = writeBuilder.newWrite(); - write.write(GenericRow.of(row[0], row[1])); - BatchTableCommit commit = writeBuilder.newCommit(); - commit.commit(write.prepareCommit()); - write.close(); - commit.close(); + commitRows(table, GenericRow.of(row[0], row[1])); } - Map masking = new HashMap<>(); - masking.put("amount", new FieldTransform(new FieldRef(1, "src", DataTypes.INT()))); - setColumnMasking(identifier, masking); + setColumnMask( + identifier, "amount", new FieldTransform(new FieldRef(1, "src", DataTypes.INT()))); - LeafPredicate amountFilter = - LeafPredicate.of( - new FieldTransform(new FieldRef(0, "amount", DataTypes.INT())), - GreaterThan.INSTANCE, - Collections.singletonList(500)); + LeafPredicate amountFilter = leaf(0, "amount", DataTypes.INT(), GreaterThan.INSTANCE, 500); ReadBuilder readBuilder = table.newReadBuilder().withProjection(new int[] {0}).withFilter(amountFilter); TableRead read = readBuilder.newRead(); @@ -4854,23 +4826,12 @@ void testStreamScanFilterOnMaskedColumnNotPushedToStats() throws Exception { Collections.singletonMap( CoreOptions.SOURCE_SPLIT_TARGET_SIZE.key(), "1 b")); for (int[] row : new int[][] {{1, 1000}, {900, 10}}) { - BatchWriteBuilder writeBuilder = table.newBatchWriteBuilder(); - BatchTableWrite write = writeBuilder.newWrite(); - write.write(GenericRow.of(row[0], row[1])); - BatchTableCommit commit = writeBuilder.newCommit(); - commit.commit(write.prepareCommit()); - write.close(); - commit.close(); + commitRows(table, GenericRow.of(row[0], row[1])); } - Map masking = new HashMap<>(); - masking.put("amount", new FieldTransform(new FieldRef(1, "src", DataTypes.INT()))); - setColumnMasking(identifier, masking); + setColumnMask( + identifier, "amount", new FieldTransform(new FieldRef(1, "src", DataTypes.INT()))); - LeafPredicate amountFilter = - LeafPredicate.of( - new FieldTransform(new FieldRef(0, "amount", DataTypes.INT())), - GreaterThan.INSTANCE, - Collections.singletonList(500)); + LeafPredicate amountFilter = leaf(0, "amount", DataTypes.INT(), GreaterThan.INSTANCE, 500); ReadBuilder readBuilder = table.newReadBuilder().withProjection(new int[] {0}).withFilter(amountFilter); StreamTableScan scan = readBuilder.newStreamScan(); @@ -4895,10 +4856,12 @@ void testMaskGrowthOnPushedFilterColumn() throws Exception { writeStringRow(table, "d1", "o1"); LeafPredicate displayFilter = - LeafPredicate.of( - new FieldTransform(new FieldRef(0, "display", DataTypes.STRING())), + leaf( + 0, + "display", + DataTypes.STRING(), Equal.INSTANCE, - Collections.singletonList(BinaryString.fromString("d1"))); + BinaryString.fromString("d1")); ReadBuilder readBuilder = table.newReadBuilder().withProjection(new int[] {0}).withFilter(displayFilter); TableScan scan = readBuilder.newScan(); @@ -4909,11 +4872,7 @@ void testMaskGrowthOnPushedFilterColumn() throws Exception { table.rowType().project("display")); assertThat(plain).hasSize(1); - Map masking = new HashMap<>(); - masking.put( - "display", - new ConcatTransform(Collections.singletonList(BinaryString.fromString("****")))); - setColumnMasking(identifier, masking); + maskConstant(identifier, "display"); assertThatThrownBy(scan::plan) .isInstanceOf(IllegalStateException.class) @@ -4946,10 +4905,7 @@ void testDeferredFilterAppliesToPartitionListing() throws Exception { writeStringRow(table, "b", "v2"); LeafPredicate partitionFilter = - LeafPredicate.of( - new FieldTransform(new FieldRef(0, "p", DataTypes.STRING())), - Equal.INSTANCE, - Collections.singletonList(BinaryString.fromString("a"))); + leaf(0, "p", DataTypes.STRING(), Equal.INSTANCE, BinaryString.fromString("a")); assertThat( table.newReadBuilder() .withFilter(partitionFilter) @@ -4972,10 +4928,7 @@ void testRowFilterAppliesToPartitionListing() throws Exception { writeStringRows(table, new String[] {"a", "v1"}, new String[] {"b", "v2"}); LeafPredicate onA = - LeafPredicate.of( - new FieldTransform(new FieldRef(0, "p", DataTypes.STRING())), - Equal.INSTANCE, - Collections.singletonList(BinaryString.fromString("a"))); + leaf(0, "p", DataTypes.STRING(), Equal.INSTANCE, BinaryString.fromString("a")); setRowFilter(identifier, Collections.singletonList(onA)); List entries = @@ -4997,23 +4950,12 @@ void testColumnMaskingDisablesLimitPushdown() throws Exception { Collections.singletonMap( CoreOptions.SOURCE_SPLIT_TARGET_SIZE.key(), "1 b")); for (int[] row : new int[][] {{900, 10}, {1, 1000}}) { - BatchWriteBuilder writeBuilder = table.newBatchWriteBuilder(); - BatchTableWrite write = writeBuilder.newWrite(); - write.write(GenericRow.of(row[0], row[1])); - BatchTableCommit commit = writeBuilder.newCommit(); - commit.commit(write.prepareCommit()); - write.close(); - commit.close(); + commitRows(table, GenericRow.of(row[0], row[1])); } - Map masking = new HashMap<>(); - masking.put("amount", new FieldTransform(new FieldRef(1, "src", DataTypes.INT()))); - setColumnMasking(identifier, masking); + setColumnMask( + identifier, "amount", new FieldTransform(new FieldRef(1, "src", DataTypes.INT()))); - LeafPredicate amountFilter = - LeafPredicate.of( - new FieldTransform(new FieldRef(0, "amount", DataTypes.INT())), - GreaterThan.INSTANCE, - Collections.singletonList(500)); + LeafPredicate amountFilter = leaf(0, "amount", DataTypes.INT(), GreaterThan.INSTANCE, 500); ReadBuilder readBuilder = table.newReadBuilder() .withProjection(new int[] {0}) @@ -5044,25 +4986,18 @@ void testFilterOnMaskedPartitionColumn() throws Exception { writeStringRow(table, "a", "va"); writeStringRow(table, "b", "vb"); - Map masking = new HashMap<>(); - masking.put("p", new FieldTransform(new FieldRef(1, "v", DataTypes.STRING()))); - setColumnMasking(identifier, masking); + setColumnMask( + identifier, "p", new FieldTransform(new FieldRef(1, "v", DataTypes.STRING()))); LeafPredicate maskedMatch = - LeafPredicate.of( - new FieldTransform(new FieldRef(0, "p", DataTypes.STRING())), - Equal.INSTANCE, - Collections.singletonList(BinaryString.fromString("vb"))); + leaf(0, "p", DataTypes.STRING(), Equal.INSTANCE, BinaryString.fromString("vb")); List rows = readWithFilter(table, maskedMatch, "p", "v"); assertThat(rows).hasSize(1); assertThat(rows.get(0).getString(0).toString()).isEqualTo("vb"); assertThat(rows.get(0).getString(1).toString()).isEqualTo("vb"); LeafPredicate rawMatch = - LeafPredicate.of( - new FieldTransform(new FieldRef(0, "p", DataTypes.STRING())), - Equal.INSTANCE, - Collections.singletonList(BinaryString.fromString("a"))); + leaf(0, "p", DataTypes.STRING(), Equal.INSTANCE, BinaryString.fromString("a")); assertThat(readWithFilter(table, rawMatch, "p", "v")).isEmpty(); List unprojected = readWithFilter(table, maskedMatch, "v"); @@ -5083,15 +5018,11 @@ void testLimitWithFilterOnMaskedPartitionColumn() throws Exception { Collections.emptyMap()); writeStringRow(table, "a", "va"); writeStringRow(table, "b", "vb"); - Map masking = new HashMap<>(); - masking.put("p", new FieldTransform(new FieldRef(1, "v", DataTypes.STRING()))); - setColumnMasking(identifier, masking); + setColumnMask( + identifier, "p", new FieldTransform(new FieldRef(1, "v", DataTypes.STRING()))); LeafPredicate maskedMatch = - LeafPredicate.of( - new FieldTransform(new FieldRef(0, "p", DataTypes.STRING())), - Equal.INSTANCE, - Collections.singletonList(BinaryString.fromString("vb"))); + leaf(0, "p", DataTypes.STRING(), Equal.INSTANCE, BinaryString.fromString("vb")); ReadBuilder readBuilder = table.newReadBuilder() .withProjection(new int[] {0, 1}) @@ -5118,24 +5049,12 @@ void testMaskedPkFilterNotAppliedOnRawValues() throws Exception { Collections.emptyList(), Collections.singletonList("id"), Collections.singletonMap(CoreOptions.BUCKET.key(), "1")); - BatchWriteBuilder writeBuilder = table.newBatchWriteBuilder(); - BatchTableWrite write = writeBuilder.newWrite(); - write.write(GenericRow.of(1, 500)); - write.write(GenericRow.of(2, 600)); - BatchTableCommit commit = writeBuilder.newCommit(); - commit.commit(write.prepareCommit()); - write.close(); - commit.close(); + commitRows(table, GenericRow.of(1, 500), GenericRow.of(2, 600)); - Map masking = new HashMap<>(); - masking.put("id", new FieldTransform(new FieldRef(1, "src", DataTypes.INT()))); - setColumnMasking(identifier, masking); + setColumnMask( + identifier, "id", new FieldTransform(new FieldRef(1, "src", DataTypes.INT()))); - LeafPredicate idFilter = - LeafPredicate.of( - new FieldTransform(new FieldRef(0, "id", DataTypes.INT())), - Equal.INSTANCE, - Collections.singletonList(500)); + LeafPredicate idFilter = leaf(0, "id", DataTypes.INT(), Equal.INSTANCE, 500); List rows = readWithFilter(table, idFilter, "id", "src"); assertThat(rows).hasSize(1); assertThat(rows.get(0).getInt(0)).isEqualTo(500); @@ -5604,21 +5523,9 @@ void testRowFilterReadLimitSkippedWithTopN() throws Exception { @Test void testColumnMaskingOrFilterWithUnprojectedOperand() throws Exception { Identifier identifier = Identifier.create("test_table_db", "auth_or_filter_unprojected"); - catalog.createDatabase(identifier.getDatabaseName(), true); - List fields = new ArrayList<>(); - fields.add(new DataField(0, "a", DataTypes.STRING())); - fields.add(new DataField(1, "b", DataTypes.STRING())); - fields.add(new DataField(2, "c", DataTypes.STRING())); - catalog.createTable( - identifier, - new Schema( - fields, - Collections.emptyList(), - Collections.emptyList(), - Collections.singletonMap(QUERY_AUTH_ENABLED.key(), "true"), - ""), - true); - Table table = catalog.getTable(identifier); + Table table = + createMaskingAuthTable( + identifier, stringFields("a", "b", "c"), Collections.emptyMap()); commitRows( table, GenericRow.of( @@ -5626,22 +5533,15 @@ void testColumnMaskingOrFilterWithUnprojectedOperand() throws Exception { BinaryString.fromString("bee"), BinaryString.fromString("cee"))); - Map masking = new HashMap<>(); - masking.put( + setColumnMask( + identifier, "a", new ConcatTransform(Collections.singletonList(BinaryString.fromString("MASKED")))); - setColumnMasking(identifier, masking); Predicate onMasked = - LeafPredicate.of( - new FieldTransform(new FieldRef(0, "a", DataTypes.STRING())), - Equal.INSTANCE, - Collections.singletonList(BinaryString.fromString("MASKED"))); + leaf(0, "a", DataTypes.STRING(), Equal.INSTANCE, BinaryString.fromString("MASKED")); Predicate onPlain = - LeafPredicate.of( - new FieldTransform(new FieldRef(1, "b", DataTypes.STRING())), - Equal.INSTANCE, - Collections.singletonList(BinaryString.fromString("bee"))); + leaf(1, "b", DataTypes.STRING(), Equal.INSTANCE, BinaryString.fromString("bee")); Predicate disjunction = PredicateBuilder.or(onMasked, onPlain); ReadBuilder readBuilder = @@ -5657,35 +5557,24 @@ void testColumnMaskingOrFilterWithUnprojectedOperand() throws Exception { @Test void testColumnMaskingPartitionListingBeforePlan() throws Exception { Identifier identifier = Identifier.create("test_table_db", "auth_partition_listing"); - catalog.createDatabase(identifier.getDatabaseName(), true); - List fields = new ArrayList<>(); - fields.add(new DataField(0, "pt", DataTypes.STRING())); - fields.add(new DataField(1, "a", DataTypes.STRING())); - catalog.createTable( - identifier, - new Schema( - fields, + Table table = + createMaskingAuthTable( + identifier, + stringFields("pt", "a"), Collections.singletonList("pt"), Collections.emptyList(), - Collections.singletonMap(QUERY_AUTH_ENABLED.key(), "true"), - ""), - true); - Table table = catalog.getTable(identifier); + Collections.emptyMap()); commitRows( table, GenericRow.of(BinaryString.fromString("p1"), BinaryString.fromString("raw"))); - Map masking = new HashMap<>(); - masking.put( + setColumnMask( + identifier, "a", new ConcatTransform(Collections.singletonList(BinaryString.fromString("MASKED")))); - setColumnMasking(identifier, masking); Predicate onMasked = - LeafPredicate.of( - new FieldTransform(new FieldRef(1, "a", DataTypes.STRING())), - Equal.INSTANCE, - Collections.singletonList(BinaryString.fromString("MASKED"))); + leaf(1, "a", DataTypes.STRING(), Equal.INSTANCE, BinaryString.fromString("MASKED")); InnerTableScan scan = (InnerTableScan) table.newReadBuilder().withFilter(onMasked).newScan(); @@ -5696,30 +5585,15 @@ void testColumnMaskingPartitionListingBeforePlan() throws Exception { @Test void testQueryAuthLimitDoesNotCutRowsBeforeFilter() throws Exception { Identifier identifier = Identifier.create("test_table_db", "auth_limit_with_filter"); - catalog.createDatabase(identifier.getDatabaseName(), true); - List fields = new ArrayList<>(); - fields.add(new DataField(0, "a", DataTypes.STRING())); - fields.add(new DataField(1, "b", DataTypes.STRING())); - catalog.createTable( - identifier, - new Schema( - fields, - Collections.emptyList(), - Collections.emptyList(), - Collections.singletonMap(QUERY_AUTH_ENABLED.key(), "true"), - ""), - true); - Table table = catalog.getTable(identifier); + Table table = + createMaskingAuthTable(identifier, stringFields("a", "b"), Collections.emptyMap()); commitRows( table, GenericRow.of(BinaryString.fromString("no"), BinaryString.fromString("r1")), GenericRow.of(BinaryString.fromString("yes"), BinaryString.fromString("r2"))); Predicate onA = - LeafPredicate.of( - new FieldTransform(new FieldRef(0, "a", DataTypes.STRING())), - Equal.INSTANCE, - Collections.singletonList(BinaryString.fromString("yes"))); + leaf(0, "a", DataTypes.STRING(), Equal.INSTANCE, BinaryString.fromString("yes")); ReadBuilder readBuilder = table.newReadBuilder().withFilter(onA).withLimit(1); List out = new ArrayList<>(); try (RecordReader reader = @@ -5742,15 +5616,11 @@ void testMaskedPartitionKeyRejectsPartitionFilter() throws Exception { writeStringRow(table, "a", "va"); writeStringRow(table, "b", "vb"); - Map masking = new HashMap<>(); - masking.put("p", new FieldTransform(new FieldRef(1, "v", DataTypes.STRING()))); - setColumnMasking(identifier, masking); + setColumnMask( + identifier, "p", new FieldTransform(new FieldRef(1, "v", DataTypes.STRING()))); LeafPredicate maskedMatch = - LeafPredicate.of( - new FieldTransform(new FieldRef(0, "p", DataTypes.STRING())), - Equal.INSTANCE, - Collections.singletonList(BinaryString.fromString("vb"))); + leaf(0, "p", DataTypes.STRING(), Equal.INSTANCE, BinaryString.fromString("vb")); InnerTableScan partitionScan = (InnerTableScan) table.newReadBuilder().newScan(); partitionScan.withPartitionFilter(maskedMatch); @@ -5773,24 +5643,17 @@ void testPartitionFilterAllowedOnUnmaskedPartitionKey() throws Exception { Collections.emptyMap()); writeStringRows(table, new String[] {"x", "a", "v1"}, new String[] {"y", "b", "v2"}); - Map masking = new HashMap<>(); - masking.put("p2", new FieldTransform(new FieldRef(2, "v", DataTypes.STRING()))); - setColumnMasking(identifier, masking); + setColumnMask( + identifier, "p2", new FieldTransform(new FieldRef(2, "v", DataTypes.STRING()))); LeafPredicate onP1 = - LeafPredicate.of( - new FieldTransform(new FieldRef(0, "p1", DataTypes.STRING())), - Equal.INSTANCE, - Collections.singletonList(BinaryString.fromString("x"))); + leaf(0, "p1", DataTypes.STRING(), Equal.INSTANCE, BinaryString.fromString("x")); InnerTableScan okScan = (InnerTableScan) table.newReadBuilder().newScan(); okScan.withPartitionFilter(onP1); assertThat(okScan.plan().splits()).isNotEmpty(); LeafPredicate onP2 = - LeafPredicate.of( - new FieldTransform(new FieldRef(1, "p2", DataTypes.STRING())), - Equal.INSTANCE, - Collections.singletonList(BinaryString.fromString("v1"))); + leaf(1, "p2", DataTypes.STRING(), Equal.INSTANCE, BinaryString.fromString("v1")); InnerTableScan badScan = (InnerTableScan) table.newReadBuilder().newScan(); badScan.withPartitionFilter(onP2); assertThatThrownBy(badScan::plan) @@ -5811,10 +5674,12 @@ void testExplicitPartitionFilterSurvivesDeferredQueryFilter() throws Exception { writeStringRows(table, new String[] {"a", "va"}, new String[] {"b", "vb"}); LeafPredicate pAtLeastA = - LeafPredicate.of( - new FieldTransform(new FieldRef(0, "p", DataTypes.STRING())), + leaf( + 0, + "p", + DataTypes.STRING(), GreaterOrEqual.INSTANCE, - Collections.singletonList(BinaryString.fromString("a"))); + BinaryString.fromString("a")); assertThat(readPartitionB(table, pAtLeastA)).containsExactly("b"); Identifier plain = Identifier.create("test_table_db", "plain_part_filter_with_filter"); @@ -5849,20 +5714,8 @@ private static List readPartitionB(Table table, Predicate filter) throws @Test void testQueryAuthLimitAppliesWhenReaderExecutesFilter() throws Exception { Identifier identifier = Identifier.create("test_table_db", "auth_limit_execute_filter"); - catalog.createDatabase(identifier.getDatabaseName(), true); - List fields = new ArrayList<>(); - fields.add(new DataField(0, "a", DataTypes.STRING())); - fields.add(new DataField(1, "b", DataTypes.STRING())); - catalog.createTable( - identifier, - new Schema( - fields, - Collections.emptyList(), - Collections.emptyList(), - Collections.singletonMap(QUERY_AUTH_ENABLED.key(), "true"), - ""), - true); - Table table = catalog.getTable(identifier); + Table table = + createMaskingAuthTable(identifier, stringFields("a", "b"), Collections.emptyMap()); commitRows( table, GenericRow.of(BinaryString.fromString("yes"), BinaryString.fromString("r1")), @@ -5870,10 +5723,7 @@ void testQueryAuthLimitAppliesWhenReaderExecutesFilter() throws Exception { GenericRow.of(BinaryString.fromString("yes"), BinaryString.fromString("r3"))); Predicate onA = - LeafPredicate.of( - new FieldTransform(new FieldRef(0, "a", DataTypes.STRING())), - Equal.INSTANCE, - Collections.singletonList(BinaryString.fromString("yes"))); + leaf(0, "a", DataTypes.STRING(), Equal.INSTANCE, BinaryString.fromString("yes")); ReadBuilder readBuilder = table.newReadBuilder().withFilter(onA).withLimit(2); List out = new ArrayList<>(); try (RecordReader reader = @@ -5898,20 +5748,13 @@ void testPartitionFilterFieldsReplacedOnReapply() throws Exception { Collections.emptyMap()); writeStringRows(table, new String[] {"x", "a", "v1"}); - Map masking = new HashMap<>(); - masking.put("p2", new FieldTransform(new FieldRef(2, "v", DataTypes.STRING()))); - setColumnMasking(identifier, masking); + setColumnMask( + identifier, "p2", new FieldTransform(new FieldRef(2, "v", DataTypes.STRING()))); LeafPredicate onMaskedP2 = - LeafPredicate.of( - new FieldTransform(new FieldRef(1, "p2", DataTypes.STRING())), - Equal.INSTANCE, - Collections.singletonList(BinaryString.fromString("v1"))); + leaf(1, "p2", DataTypes.STRING(), Equal.INSTANCE, BinaryString.fromString("v1")); LeafPredicate onPlainP1 = - LeafPredicate.of( - new FieldTransform(new FieldRef(0, "p1", DataTypes.STRING())), - Equal.INSTANCE, - Collections.singletonList(BinaryString.fromString("x"))); + leaf(0, "p1", DataTypes.STRING(), Equal.INSTANCE, BinaryString.fromString("x")); InnerTableScan scan = (InnerTableScan) table.newReadBuilder().newScan(); scan.withPartitionFilter(onMaskedP2); @@ -5926,11 +5769,7 @@ void testPhysicalMetadataSystemTablesRejectedUnderQueryAuth() throws Exception { createMaskingAuthTable( identifier, stringFields("secret", "other"), Collections.emptyMap()); writeStringRow(table, "TOPSECRET", "o1"); - Map masking = new HashMap<>(); - masking.put( - "secret", - new ConcatTransform(Collections.singletonList(BinaryString.fromString("****")))); - setColumnMasking(identifier, masking); + maskConstant(identifier, "secret"); for (String suffix : Arrays.asList("files", "file_key_ranges", "binlog", "statistics")) { Identifier sysId =