From 8505418b71c312328d900df1320eeae89d7a78c2 Mon Sep 17 00:00:00 2001 From: JingsongLi Date: Wed, 26 Aug 2026 16:53:15 +0800 Subject: [PATCH 1/5] [core] Fail closed for managed query policies --- .../paimon/catalog/TableQueryAuthResult.java | 42 ++++----- .../table/source/AbstractDataTableRead.java | 19 +++- .../catalog/TableQueryAuthResultTest.java | 87 +++++++++++++++++++ 3 files changed, 126 insertions(+), 22 deletions(-) create mode 100644 paimon-core/src/test/java/org/apache/paimon/catalog/TableQueryAuthResultTest.java diff --git a/paimon-core/src/main/java/org/apache/paimon/catalog/TableQueryAuthResult.java b/paimon-core/src/main/java/org/apache/paimon/catalog/TableQueryAuthResult.java index 0524eea1b07c..6d381078c739 100644 --- a/paimon-core/src/main/java/org/apache/paimon/catalog/TableQueryAuthResult.java +++ b/paimon-core/src/main/java/org/apache/paimon/catalog/TableQueryAuthResult.java @@ -48,6 +48,8 @@ import java.util.TreeMap; import java.util.stream.Collectors; +import static org.apache.paimon.utils.Preconditions.checkArgument; + /** Auth result for table query, including row level filter and optional column masking rules. */ public class TableQueryAuthResult implements Serializable { @@ -89,13 +91,10 @@ public Predicate extractPredicate() { if (filter != null && !filter.isEmpty()) { List predicates = new ArrayList<>(); for (String json : filter) { - if (StringUtils.isEmpty(json)) { - continue; - } + checkArgument(!StringUtils.isEmpty(json), "Row filter cannot be empty."); Predicate predicate = JsonSerdeUtil.fromJson(json, Predicate.class); - if (predicate != null) { - predicates.add(predicate); - } + checkArgument(predicate != null, "Row filter cannot be JSON null."); + predicates.add(predicate); } if (predicates.size() == 1) { rowFilter = predicates.get(0); @@ -122,13 +121,10 @@ public Map extractColumnMasking() { for (Map.Entry e : columnMasking.entrySet()) { String column = e.getKey(); String json = e.getValue(); - if (StringUtils.isEmpty(column) || StringUtils.isEmpty(json)) { - continue; - } + checkArgument(!StringUtils.isEmpty(column), "Column mask target cannot be empty."); + checkArgument(!StringUtils.isEmpty(json), "Column mask transform cannot be empty."); Transform transform = JsonSerdeUtil.fromJson(json, Transform.class); - if (transform == null) { - continue; - } + checkArgument(transform != null, "Column mask transform cannot be JSON null."); result.put(column, transform); } } @@ -184,14 +180,15 @@ private static Map transformRemapping( for (Map.Entry e : masking.entrySet()) { String targetColumn = e.getKey(); Transform transform = e.getValue(); - if (targetColumn == null || transform == null) { - continue; - } + checkArgument(targetColumn != null, "Column mask target cannot be null."); + checkArgument(transform != null, "Column mask transform cannot be null."); int targetIndex = outputRowType.getFieldIndex(targetColumn); - if (targetIndex < 0) { - continue; - } + checkArgument( + targetIndex >= 0, + "Column mask target '%s' is not present in output row type %s.", + targetColumn, + outputRowType); List newInputs = new ArrayList<>(); for (Object input : transform.inputs()) { @@ -234,7 +231,7 @@ public Predicate visit(LeafPredicate predicate) { String fieldName = ref.name(); int newIndex = outputRowType.getFieldIndex(fieldName); if (newIndex < 0) { - throw new RuntimeException( + throw new IllegalArgumentException( String.format( "Unable to read data without column %s when row filter enabled.", fieldName)); @@ -250,15 +247,20 @@ public Predicate visit(LeafPredicate predicate) { @Override public Predicate visit(CompoundPredicate predicate) { + checkArgument( + predicate.function() != null, "Compound row filter function cannot be null."); + checkArgument( + predicate.children() != null, "Compound row filter children cannot be null."); List remappedChildren = new ArrayList<>(); for (Predicate child : predicate.children()) { + checkArgument(child != null, "Compound row filter child cannot be null."); Predicate remapped = child.visit(this); if (remapped != null) { remappedChildren.add(remapped); } } if (remappedChildren.isEmpty()) { - return null; + throw new IllegalArgumentException("Compound row filter must contain a predicate."); } if (remappedChildren.size() == 1) { return remappedChildren.get(0); diff --git a/paimon-core/src/main/java/org/apache/paimon/table/source/AbstractDataTableRead.java b/paimon-core/src/main/java/org/apache/paimon/table/source/AbstractDataTableRead.java index 59e8cc0666ee..6d6f572d2b12 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/source/AbstractDataTableRead.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/source/AbstractDataTableRead.java @@ -21,8 +21,10 @@ import org.apache.paimon.catalog.TableQueryAuthResult; import org.apache.paimon.data.InternalRow; import org.apache.paimon.disk.IOManager; +import org.apache.paimon.predicate.FieldRef; import org.apache.paimon.predicate.Predicate; import org.apache.paimon.predicate.PredicateProjectionConverter; +import org.apache.paimon.predicate.Transform; import org.apache.paimon.reader.RecordReader; import org.apache.paimon.schema.TableSchema; import org.apache.paimon.types.RowType; @@ -35,6 +37,7 @@ import java.util.ArrayList; import java.util.HashSet; import java.util.List; +import java.util.Map; import java.util.Optional; import java.util.Set; @@ -138,9 +141,21 @@ private RecordReader authedReader(Split split, TableQueryAuthResult RowType tableType = schema.logicalRowType(); RowType readType = this.readType == null ? tableType : this.readType; Predicate authPredicate = authResult.extractPredicate(); + Map columnMasking = authResult.extractColumnMasking(); ProjectedRow backRow = null; + Set authFields = new HashSet<>(); if (authPredicate != null) { - Set authFields = collectFieldNames(authPredicate); + authFields.addAll(collectFieldNames(authPredicate)); + } + for (Map.Entry mask : columnMasking.entrySet()) { + authFields.add(mask.getKey()); + for (Object input : mask.getValue().inputs()) { + if (input instanceof FieldRef) { + authFields.add(((FieldRef) input).name()); + } + } + } + if (!authFields.isEmpty()) { List readFields = readType.getFieldNames(); List authAddNames = new ArrayList<>(); Set readFieldSet = new HashSet<>(readFields); @@ -151,7 +166,7 @@ private RecordReader authedReader(Split split, TableQueryAuthResult } if (!authAddNames.isEmpty()) { readType = tableType.project(ListUtils.union(readFields, authAddNames)); - withReadType(readType); + applyReadType(readType); backRow = ProjectedRow.from(readType.projectIndexes(readFields)); } } diff --git a/paimon-core/src/test/java/org/apache/paimon/catalog/TableQueryAuthResultTest.java b/paimon-core/src/test/java/org/apache/paimon/catalog/TableQueryAuthResultTest.java new file mode 100644 index 000000000000..3d45924871f1 --- /dev/null +++ b/paimon-core/src/test/java/org/apache/paimon/catalog/TableQueryAuthResultTest.java @@ -0,0 +1,87 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.catalog; + +import org.apache.paimon.predicate.Predicate; +import org.apache.paimon.types.RowType; +import org.apache.paimon.utils.JsonSerdeUtil; + +import org.junit.jupiter.api.Test; + +import java.util.Collections; + +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/** Tests that malformed query-authorization definitions cannot be silently ignored. */ +public class TableQueryAuthResultTest { + + @Test + void testInvalidRowFilterFailsClosed() { + assertThatThrownBy( + () -> + new TableQueryAuthResult(Collections.singletonList(""), null) + .extractPredicate()) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("cannot be empty"); + assertThatThrownBy( + () -> + new TableQueryAuthResult(Collections.singletonList("null"), null) + .extractPredicate()) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("JSON null"); + + Predicate emptyCompound = + JsonSerdeUtil.fromJson( + "{\"kind\":\"COMPOUND\",\"function\":\"AND\",\"children\":[]}", + Predicate.class); + assertThatThrownBy(() -> TableQueryAuthResult.remapPredicate(emptyCompound, RowType.of())) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("must contain a predicate"); + + Predicate missingFunction = + JsonSerdeUtil.fromJson( + "{\"kind\":\"COMPOUND\",\"function\":null,\"children\":[" + + "{\"kind\":\"LEAF\",\"transform\":{\"name\":\"NULL\"}," + + "\"function\":\"TRUE\",\"literals\":[]}," + + "{\"kind\":\"LEAF\",\"transform\":{\"name\":\"NULL\"}," + + "\"function\":\"TRUE\",\"literals\":[]}]}", + Predicate.class); + assertThatThrownBy(() -> TableQueryAuthResult.remapPredicate(missingFunction, RowType.of())) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("function cannot be null"); + } + + @Test + void testInvalidColumnMaskFailsClosed() { + assertThatThrownBy( + () -> + new TableQueryAuthResult( + null, Collections.singletonMap("email", "")) + .extractColumnMasking()) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("cannot be empty"); + assertThatThrownBy( + () -> + new TableQueryAuthResult( + null, Collections.singletonMap("email", "null")) + .extractColumnMasking()) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("JSON null"); + } +} From 79bcb5fccfa1b11b7e7c9ea95a5cc6f03a2586a0 Mon Sep 17 00:00:00 2001 From: JingsongLi Date: Wed, 26 Aug 2026 16:54:42 +0800 Subject: [PATCH 2/5] [test] Add REST management test service --- .../org/apache/paimon/rest/PolicyKey.java | 90 ++ .../org/apache/paimon/rest/PolicyKeyTest.java | 56 + .../apache/paimon/rest/RESTCatalogServer.java | 960 ++++++++++++++++-- .../rest/RESTColumnPermissionSupport.java | 135 +++ .../paimon/rest/RESTPermissionStore.java | 114 +++ .../paimon/rest/RESTPermissionStoreTest.java | 123 +++ .../paimon/rest/TableLifecycleLocks.java | 38 + 7 files changed, 1418 insertions(+), 98 deletions(-) create mode 100644 paimon-core/src/test/java/org/apache/paimon/rest/PolicyKey.java create mode 100644 paimon-core/src/test/java/org/apache/paimon/rest/PolicyKeyTest.java create mode 100644 paimon-core/src/test/java/org/apache/paimon/rest/RESTColumnPermissionSupport.java create mode 100644 paimon-core/src/test/java/org/apache/paimon/rest/RESTPermissionStore.java create mode 100644 paimon-core/src/test/java/org/apache/paimon/rest/RESTPermissionStoreTest.java create mode 100644 paimon-core/src/test/java/org/apache/paimon/rest/TableLifecycleLocks.java diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/PolicyKey.java b/paimon-core/src/test/java/org/apache/paimon/rest/PolicyKey.java new file mode 100644 index 000000000000..325c1bceb2ad --- /dev/null +++ b/paimon-core/src/test/java/org/apache/paimon/rest/PolicyKey.java @@ -0,0 +1,90 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.rest; + +import org.apache.paimon.management.DataPolicy; +import org.apache.paimon.management.PolicyType; + +import javax.annotation.Nullable; + +import java.util.Objects; + +/** Stable identity of a policy stored by the REST catalog test server. */ +final class PolicyKey implements Comparable { + + final String tableUuid; + final PolicyType type; + final String principal; + @Nullable final String column; + + PolicyKey(String tableUuid, DataPolicy policy) { + this( + tableUuid, + policy.type(), + policy.getPrincipal(), + policy.getColumnMask() == null ? null : policy.getColumnMask().getOnColumn()); + } + + PolicyKey(String tableUuid, PolicyType type, String principal, @Nullable String column) { + this.tableUuid = tableUuid; + this.type = type; + this.principal = principal; + this.column = column; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof PolicyKey)) { + return false; + } + PolicyKey that = (PolicyKey) o; + return tableUuid.equals(that.tableUuid) + && type == that.type + && principal.equals(that.principal) + && Objects.equals(column, that.column); + } + + @Override + public int hashCode() { + return Objects.hash(tableUuid, type, principal, column); + } + + @Override + public int compareTo(PolicyKey that) { + int result = tableUuid.compareTo(that.tableUuid); + if (result != 0) { + return result; + } + result = type.compareTo(that.type); + if (result != 0) { + return result; + } + result = principal.compareTo(that.principal); + if (result != 0) { + return result; + } + if (column == null) { + return that.column == null ? 0 : -1; + } + return that.column == null ? 1 : column.compareTo(that.column); + } +} diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/PolicyKeyTest.java b/paimon-core/src/test/java/org/apache/paimon/rest/PolicyKeyTest.java new file mode 100644 index 000000000000..027736ded674 --- /dev/null +++ b/paimon-core/src/test/java/org/apache/paimon/rest/PolicyKeyTest.java @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.rest; + +import org.apache.paimon.management.ColumnMask; +import org.apache.paimon.management.DataPolicy; +import org.apache.paimon.management.PermissionResource; +import org.apache.paimon.management.ResourceType; + +import org.junit.jupiter.api.Test; + +import java.util.TreeSet; + +import static org.assertj.core.api.Assertions.assertThat; + +/** Tests for policy identities used by the REST catalog test server. */ +public class PolicyKeyTest { + + @Test + void testOrderingDoesNotFlattenOpaquePrincipalAndColumn() { + PermissionResource resource = + new PermissionResource(ResourceType.TABLE, "database", "table", null, null); + PolicyKey first = + new PolicyKey( + "table-id", + DataPolicy.columnMask( + resource, new ColumnMask("c", "{\"name\":\"NULL\"}"), "a:b")); + PolicyKey second = + new PolicyKey( + "table-id", + DataPolicy.columnMask( + resource, new ColumnMask("b:c", "{\"name\":\"NULL\"}"), "a")); + + TreeSet sorted = new TreeSet<>(); + sorted.add(first); + sorted.add(second); + + assertThat(sorted).containsExactly(second, first); + } +} diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogServer.java b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogServer.java index 733b4b1c9fef..b6e1270cb6e9 100644 --- a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogServer.java +++ b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogServer.java @@ -40,11 +40,19 @@ import org.apache.paimon.function.FunctionChange; import org.apache.paimon.function.FunctionDefinition; import org.apache.paimon.function.FunctionImpl; +import org.apache.paimon.management.ColumnMask; +import org.apache.paimon.management.DataPolicy; +import org.apache.paimon.management.ListPermissionsRequest; +import org.apache.paimon.management.PermissionAssignment; +import org.apache.paimon.management.PermissionResource; +import org.apache.paimon.management.ResourceType; +import org.apache.paimon.management.RowFilter; import org.apache.paimon.operation.Lock; import org.apache.paimon.options.Options; import org.apache.paimon.partition.Partition; import org.apache.paimon.partition.PartitionStatistics; import org.apache.paimon.partition.PartitionUtils; +import org.apache.paimon.predicate.FieldRef; import org.apache.paimon.predicate.Predicate; import org.apache.paimon.predicate.Transform; import org.apache.paimon.rest.auth.AuthProvider; @@ -63,12 +71,16 @@ import org.apache.paimon.rest.requests.CreateTagRequest; import org.apache.paimon.rest.requests.CreateViewRequest; import org.apache.paimon.rest.requests.DropPartitionsRequest; +import org.apache.paimon.rest.requests.DropPolicyRequest; +import org.apache.paimon.rest.requests.GrantPermissionRequest; import org.apache.paimon.rest.requests.ListPartitionsByFilterRequest; import org.apache.paimon.rest.requests.ListPartitionsByNamesRequest; import org.apache.paimon.rest.requests.MarkDonePartitionsRequest; +import org.apache.paimon.rest.requests.PolicyRequest; import org.apache.paimon.rest.requests.RenameTableRequest; import org.apache.paimon.rest.requests.ReplaceTableRequest; import org.apache.paimon.rest.requests.ResetConsumerRequest; +import org.apache.paimon.rest.requests.RevokePermissionRequest; import org.apache.paimon.rest.requests.RollbackSchemaRequest; import org.apache.paimon.rest.requests.RollbackTableRequest; import org.apache.paimon.rest.responses.AlterDatabaseResponse; @@ -93,6 +105,8 @@ import org.apache.paimon.rest.responses.ListFunctionsGloballyResponse; import org.apache.paimon.rest.responses.ListFunctionsResponse; import org.apache.paimon.rest.responses.ListPartitionsResponse; +import org.apache.paimon.rest.responses.ListPermissionsResponse; +import org.apache.paimon.rest.responses.ListPoliciesResponse; import org.apache.paimon.rest.responses.ListSnapshotsResponse; import org.apache.paimon.rest.responses.ListTableDetailsResponse; import org.apache.paimon.rest.responses.ListTablesGloballyResponse; @@ -161,6 +175,7 @@ import java.util.Queue; import java.util.Set; import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentLinkedQueue; import java.util.regex.Pattern; import java.util.stream.Collectors; @@ -185,6 +200,7 @@ import static org.apache.paimon.rest.ResourcePaths.TABLE_DETAILS; import static org.apache.paimon.rest.ResourcePaths.VIEWS; import static org.apache.paimon.rest.ResourcePaths.VIEW_DETAILS; +import static org.apache.paimon.utils.Preconditions.checkArgument; /** Mock REST server for testing. */ public class RESTCatalogServer { @@ -195,13 +211,22 @@ public class RESTCatalogServer { public static final String AUTHORIZATION_HEADER_KEY = "Authorization"; private final String databaseUri; + private final String permissionUri; private final CatalogContext catalogContext; private final RESTFileSystemCatalog catalog; private final MockWebServer server; private final Map databaseStore = new HashMap<>(); - private final Map tableMetadataStore = new HashMap<>(); + private final Map tableMetadataStore = new ConcurrentHashMap<>(); + private final RESTPermissionStore permissionStore = new RESTPermissionStore(); + private final Map policyStore = new ConcurrentHashMap<>(); + private final Map tablePolicyLocks = new ConcurrentHashMap<>(); + private final TableLifecycleLocks tableLifecycleLocks = new TableLifecycleLocks(); + private final Set managementPrincipals = new HashSet<>(); + private final Set queryPrincipals = new HashSet<>(); + private final Set noManagementPermissionResources = + ConcurrentHashMap.newKeySet(); private final List receivedListPartitionsByFilterRequests = new java.util.concurrent.CopyOnWriteArrayList<>(); @@ -238,6 +263,7 @@ public RESTCatalogServer( this.configResponse.getDefaults().get(RESTCatalogInternalOptions.PREFIX.key()); this.resourcePaths = new ResourcePaths(prefix); this.databaseUri = resourcePaths.databases(); + this.permissionUri = resourcePaths.permissions(); Options conf = new Options(); this.configResponse.getDefaults().forEach(conf::setString); conf.setString(WAREHOUSE.key(), dataPath); @@ -332,11 +358,36 @@ public void addTableColumnAuth(Identifier identifier, List select) { } public void setRowFilterAuth(Identifier identifier, List rowFilters) { - rowFilterAuthHandler.put(identifier.getFullName(), rowFilters); + if (rowFilters == null) { + rowFilterAuthHandler.remove(identifier.getFullName()); + } else { + rowFilterAuthHandler.put(identifier.getFullName(), rowFilters); + } } public void setColumnMaskingAuth(Identifier identifier, Map columnMasking) { - columnMaskingAuthHandler.put(identifier.getFullName(), columnMasking); + if (columnMasking == null) { + columnMaskingAuthHandler.remove(identifier.getFullName()); + } else { + columnMaskingAuthHandler.put(identifier.getFullName(), columnMasking); + } + } + + public void registerManagementPrincipal(String principal) { + managementPrincipals.add(principal); + } + + public void setQueryPrincipals(Set principals) { + queryPrincipals.clear(); + queryPrincipals.addAll(principals); + } + + public void denyManagementPermission(PermissionResource resource) { + noManagementPermissionResources.add(resource); + } + + public void allowManagementPermission(PermissionResource resource) { + noManagementPermissionResources.remove(resource); } public RESTToken getDataToken(Identifier identifier) { @@ -384,6 +435,13 @@ public MockResponse dispatch(RecordedRequest request) { .queryParameter(WAREHOUSE.key()) .equals(warehouse)) { return mockResponse(configResponse, 200); + } else if (permissionUri.equals(resourcePath) + || request.getPath().startsWith(permissionUri + "/")) { + return permissionsApiHandler( + request.getMethod(), resourcePath, data, parameters); + } else if (isPolicyPath(resourcePath)) { + return policiesApiHandler( + request.getMethod(), resourcePath, data, parameters); } else if (databaseUri.equals(request.getPath()) || request.getPath().contains(databaseUri + "?")) { return databasesApiHandler(restAuthParameter.method(), data, parameters); @@ -996,6 +1054,17 @@ private MockResponse authTable(Identifier identifier, String data) throws Except if (metadata == null) { throw new Catalog.TableNotExistException(identifier); } + synchronized (policyLock(metadata.uuid())) { + TableMetadata current = tableMetadataStore.get(identifier.getFullName()); + if (current == null || !current.uuid().equals(metadata.uuid())) { + throw new Catalog.TableNotExistException(identifier); + } + return authTable(identifier, requestBody, current); + } + } + + private MockResponse authTable( + Identifier identifier, AuthTableQueryRequest requestBody, TableMetadata metadata) { List columnAuth = columnAuthHandler.get(identifier.getFullName()); if (columnAuth != null) { List select = requestBody.select(); @@ -1009,20 +1078,61 @@ private MockResponse authTable(Identifier identifier, String data) throws Except } }); } - List rowFilters = rowFilterAuthHandler.get(identifier.getFullName()); + if (!RESTColumnPermissionSupport.canSelect( + permissionStore, queryPrincipals, identifier, metadata, requestBody.select())) { + throw new Catalog.TableNoPermissionException(identifier); + } + List rowFilters = + new ArrayList<>( + rowFilterAuthHandler.getOrDefault( + identifier.getFullName(), Collections.emptyList())); Map columnMasking = - columnMaskingAuthHandler.get(identifier.getFullName()); + new HashMap<>( + columnMaskingAuthHandler.getOrDefault( + identifier.getFullName(), Collections.emptyMap())); + for (Map.Entry entry : policyStore.entrySet()) { + if (!entry.getKey().tableUuid.equals(metadata.uuid()) + || !appliesToQueryPrincipal(entry.getValue())) { + continue; + } + DataPolicy policy = entry.getValue(); + RowFilter rowFilter = policy.getRowFilter(); + if (rowFilter != null) { + Predicate predicate; + try { + predicate = parseRowFilter(metadata.schema(), rowFilter); + } catch (RuntimeException e) { + throw new Catalog.TableNoPermissionException(identifier); + } + if (predicate == null) { + throw new Catalog.TableNoPermissionException(identifier); + } + rowFilters.add(predicate); + continue; + } + ColumnMask columnMask = policy.getColumnMask(); + Transform transform; + try { + transform = parseColumnMask(metadata.schema(), columnMask); + } catch (RuntimeException e) { + throw new Catalog.TableNoPermissionException(identifier); + } + if (transform == null || columnMasking.containsKey(columnMask.getOnColumn())) { + throw new Catalog.TableNoPermissionException(identifier); + } + columnMasking.put(columnMask.getOnColumn(), transform); + } // Convert Predicate list to JSON string list List filterJsonList = null; - if (rowFilters != null) { + if (!rowFilters.isEmpty()) { filterJsonList = rowFilters.stream().map(JsonSerdeUtil::toFlatJson).collect(Collectors.toList()); } // Convert Transform map to JSON string map Map columnMaskingJsonMap = null; - if (columnMasking != null) { + if (!columnMasking.isEmpty()) { columnMaskingJsonMap = columnMasking.entrySet().stream() .collect( @@ -1039,6 +1149,10 @@ private MockResponse authTable(Identifier identifier, String data) throws Except return mockResponse(response, 200); } + private boolean appliesToQueryPrincipal(DataPolicy policy) { + return queryPrincipals.contains(policy.getPrincipal()); + } + private MockResponse commitTableHandle(Identifier identifier, String data) throws Exception { CommitTableRequest requestBody = RESTApi.fromJson(data, CommitTableRequest.class); if (noPermissionTables.contains(identifier.getFullName())) { @@ -1496,6 +1610,7 @@ private MockResponse databaseHandle(String method, String data, String databaseN return mockResponse(response, 200); case "DELETE": catalog.dropDatabase(databaseName, false, true); + removeDatabaseTableState(databaseName); databaseStore.remove(databaseName); return new MockResponse().setResponseCode(200); case "POST": @@ -1540,6 +1655,31 @@ private MockResponse databaseHandle(String method, String data, String databaseN return new MockResponse().setResponseCode(404); } + private void removeDatabaseTableState(String databaseName) { + List tableNames = + tableMetadataStore.keySet().stream() + .filter( + tableName -> + databaseName.equals( + Identifier.fromString(tableName).getDatabaseName())) + .collect(Collectors.toList()); + for (String tableName : tableNames) { + synchronized (tableLifecycleLocks.lock(tableName)) { + TableMetadata metadata = tableMetadataStore.get(tableName); + if (metadata == null) { + continue; + } + synchronized (policyLock(metadata.uuid())) { + if (tableMetadataStore.remove(tableName, metadata)) { + removePolicies(metadata.uuid()); + tableLatestSnapshotStore.remove(tableName); + tablePartitionsStore.remove(tableName); + } + } + } + } + } + private MockResponse tablesHandle( String method, String data, String databaseName, Map parameters) throws Exception { @@ -1552,25 +1692,29 @@ private MockResponse tablesHandle( CreateTableRequest requestBody = RESTApi.fromJson(data, CreateTableRequest.class); Identifier identifier = requestBody.getIdentifier(); - Schema schema = requestBody.getSchema(); - TableMetadata tableMetadata; - if (isObjectTable(schema)) { - tableMetadata = createObjectTable(identifier, schema); - } else { - catalog.createTable(identifier, schema, false); - boolean isExternal = - schema.options() != null - && schema.options().containsKey(PATH.key()); - tableMetadata = - createTableMetadata( - requestBody.getIdentifier(), - 0L, - requestBody.getSchema(), - UUID.randomUUID().toString(), - isExternal); + synchronized (tableLifecycleLocks.lock(identifier.getFullName())) { + if (tableMetadataStore.containsKey(identifier.getFullName())) { + throw new Catalog.TableAlreadyExistException(identifier); + } + Schema schema = requestBody.getSchema(); + TableMetadata tableMetadata; + if (isObjectTable(schema)) { + tableMetadata = createObjectTable(identifier, schema); + } else { + catalog.createTable(identifier, schema, false); + boolean isExternal = + schema.options() != null + && schema.options().containsKey(PATH.key()); + tableMetadata = + createTableMetadata( + requestBody.getIdentifier(), + 0L, + requestBody.getSchema(), + UUID.randomUUID().toString(), + isExternal); + } + tableMetadataStore.put(identifier.getFullName(), tableMetadata); } - tableMetadataStore.put( - requestBody.getIdentifier().getFullName(), tableMetadata); return new MockResponse().setResponseCode(200); default: return new MockResponse().setResponseCode(404); @@ -1815,20 +1959,29 @@ private MockResponse tableHandle(String method, String data, Identifier identifi alterTableImpl(identifier, requestBody.getChanges()); return new MockResponse().setResponseCode(200); case "DELETE": - if (!tableMetadataStore.containsKey(identifier.getFullName())) { - return new MockResponse().setResponseCode(404); - } - tableMetadata = tableMetadataStore.get(identifier.getFullName()); - if (!tableMetadata.isExternal()) { - try { - catalog.dropTable(identifier, false); - } catch (Exception e) { - System.out.println(e.getMessage()); + synchronized (tableLifecycleLocks.lock(identifier.getFullName())) { + tableMetadata = tableMetadataStore.get(identifier.getFullName()); + if (tableMetadata == null) { + return new MockResponse().setResponseCode(404); + } + synchronized (policyLock(tableMetadata.uuid())) { + TableMetadata current = tableMetadataStore.get(identifier.getFullName()); + if (current == null || !current.uuid().equals(tableMetadata.uuid())) { + return new MockResponse().setResponseCode(404); + } + if (!current.isExternal()) { + try { + catalog.dropTable(identifier, false); + } catch (Exception e) { + System.out.println(e.getMessage()); + } + } + removePolicies(current.uuid()); + tableMetadataStore.remove(identifier.getFullName(), current); + tableLatestSnapshotStore.remove(identifier.getFullName()); + tablePartitionsStore.remove(identifier.getFullName()); } } - tableMetadataStore.remove(identifier.getFullName()); - tableLatestSnapshotStore.remove(identifier.getFullName()); - tablePartitionsStore.remove(identifier.getFullName()); return new MockResponse().setResponseCode(200); default: return new MockResponse().setResponseCode(404); @@ -1838,32 +1991,49 @@ private MockResponse tableHandle(String method, String data, Identifier identifi private MockResponse replaceTableHandle(Identifier identifier, String data) throws Exception { ReplaceTableRequest requestBody = RESTApi.fromJson(data, ReplaceTableRequest.class); Schema newSchema = requestBody.getSchema(); - if (!tableMetadataStore.containsKey(identifier.getFullName())) { - throw new Catalog.TableNotExistException(identifier); - } TableMetadata tableMetadata = tableMetadataStore.get(identifier.getFullName()); - if (isFormatTable(tableMetadata.schema().toSchema()) || isFormatTable(newSchema)) { - throw new UnsupportedOperationException("replaceTable does not support format tables."); + if (tableMetadata == null) { + throw new Catalog.TableNotExistException(identifier); } - catalog.replaceTable(identifier, newSchema, false); - TableSchema replacedSchema = catalog.loadTableSchema(identifier); - TableMetadata newTableMetadata = - createTableMetadata( - identifier, - replacedSchema.id(), - replacedSchema.toSchema(), - tableMetadata.uuid(), - tableMetadata.isExternal()); - tableMetadataStore.put(identifier.getFullName(), newTableMetadata); - FileStoreTable table = (FileStoreTable) catalog.getTable(identifier); - Snapshot truncateSnapshot = table.snapshotManager().latestSnapshot(); - if (truncateSnapshot != null) { - tableLatestSnapshotStore.put( - identifier.getFullName(), new TableSnapshot(truncateSnapshot, 0L, 0L, 0L, 0L)); - } else { - tableLatestSnapshotStore.remove(identifier.getFullName()); + synchronized (policyLock(tableMetadata.uuid())) { + TableMetadata current = tableMetadataStore.get(identifier.getFullName()); + if (current == null || !current.uuid().equals(tableMetadata.uuid())) { + throw new Catalog.TableNotExistException(identifier); + } + TableSchema replacementSchema = + createTableMetadata( + identifier, + current.schema().id() + 1, + newSchema, + current.uuid(), + current.isExternal()) + .schema(); + validatePoliciesForSchema(identifier, current.uuid(), replacementSchema); + if (isFormatTable(current.schema().toSchema()) || isFormatTable(newSchema)) { + throw new UnsupportedOperationException( + "replaceTable does not support format tables."); + } + catalog.replaceTable(identifier, newSchema, false); + TableSchema replacedSchema = catalog.loadTableSchema(identifier); + TableMetadata newTableMetadata = + createTableMetadata( + identifier, + replacedSchema.id(), + replacedSchema.toSchema(), + current.uuid(), + current.isExternal()); + tableMetadataStore.put(identifier.getFullName(), newTableMetadata); + FileStoreTable table = (FileStoreTable) catalog.getTable(identifier); + Snapshot truncateSnapshot = table.snapshotManager().latestSnapshot(); + if (truncateSnapshot != null) { + tableLatestSnapshotStore.put( + identifier.getFullName(), + new TableSnapshot(truncateSnapshot, 0L, 0L, 0L, 0L)); + } else { + tableLatestSnapshotStore.remove(identifier.getFullName()); + } + tablePartitionsStore.remove(identifier.getFullName()); } - tablePartitionsStore.remove(identifier.getFullName()); return new MockResponse().setResponseCode(200); } @@ -1873,18 +2043,37 @@ private MockResponse renameTableHandle(String data) throws Exception { Identifier toTable = requestBody.getDestination(); if (noPermissionTables.contains(fromTable.getFullName())) { throw new Catalog.TableNoPermissionException(fromTable); - } else if (tableMetadataStore.containsKey(fromTable.getFullName())) { - TableMetadata tableMetadata = tableMetadataStore.get(fromTable.getFullName()); - if (!isFormatTable(tableMetadata.schema().toSchema()) && !tableMetadata.isExternal()) { - catalog.renameTable(requestBody.getSource(), requestBody.getDestination(), false); - } - if (tableMetadataStore.containsKey(toTable.getFullName())) { - throw new Catalog.TableAlreadyExistException(toTable); + } + Object[] locks = + tableLifecycleLocks.ordered(fromTable.getFullName(), toTable.getFullName()); + synchronized (locks[0]) { + synchronized (locks[1]) { + TableMetadata observed = tableMetadataStore.get(fromTable.getFullName()); + if (observed == null) { + throw new Catalog.TableNotExistException(fromTable); + } + synchronized (policyLock(observed.uuid())) { + TableMetadata current = tableMetadataStore.get(fromTable.getFullName()); + if (current == null || !current.uuid().equals(observed.uuid())) { + throw new Catalog.TableNotExistException(fromTable); + } + if (tableMetadataStore.containsKey(toTable.getFullName())) { + throw new Catalog.TableAlreadyExistException(toTable); + } + if (!isFormatTable(current.schema().toSchema()) && !current.isExternal()) { + catalog.renameTable(fromTable, toTable, false); + } + TableMetadata renamedMetadata = + createTableMetadata( + toTable, + current.schema().id(), + current.schema().toSchema(), + current.uuid(), + current.isExternal()); + tableMetadataStore.remove(fromTable.getFullName(), current); + tableMetadataStore.put(toTable.getFullName(), renamedMetadata); + } } - tableMetadataStore.remove(fromTable.getFullName()); - tableMetadataStore.put(toTable.getFullName(), tableMetadata); - } else { - throw new Catalog.TableNotExistException(fromTable); } return new MockResponse().setResponseCode(200); } @@ -2804,45 +2993,52 @@ private MockResponse renameViewHandle(String data) throws Exception { protected void alterTableImpl(Identifier identifier, List changes) throws Catalog.TableNotExistException, Catalog.ColumnAlreadyExistException, Catalog.ColumnNotExistException { - if (tableMetadataStore.containsKey(identifier.getFullName())) { - TableMetadata tableMetadata = tableMetadataStore.get(identifier.getFullName()); - try { - TableSchema schema = tableMetadata.schema(); - if (isFormatTable(schema.toSchema())) { - TableSchema newSchema = + TableMetadata tableMetadata = tableMetadataStore.get(identifier.getFullName()); + if (tableMetadata != null) { + synchronized (policyLock(tableMetadata.uuid())) { + TableMetadata current = tableMetadataStore.get(identifier.getFullName()); + if (current == null || !current.uuid().equals(tableMetadata.uuid())) { + throw new Catalog.TableNotExistException(identifier); + } + try { + TableSchema schema = current.schema(); + TableSchema candidateSchema = SchemaManager.generateTableSchema( schema, changes, new LazyField<>(() -> false), new LazyField<>(() -> identifier)); + validatePoliciesForSchema(identifier, current.uuid(), candidateSchema); + if (isFormatTable(schema.toSchema())) { + TableMetadata newTableMetadata = + createTableMetadata( + identifier, + candidateSchema.id(), + candidateSchema.toSchema(), + current.uuid(), + current.isExternal()); + tableMetadataStore.put(identifier.getFullName(), newTableMetadata); + return; + } + catalog.alterTable(identifier, changes, false); + FileStoreTable table = (FileStoreTable) catalog.getTable(identifier); + TableSchema newSchema = table.schema(); TableMetadata newTableMetadata = createTableMetadata( identifier, newSchema.id(), newSchema.toSchema(), - tableMetadata.uuid(), - tableMetadata.isExternal()); + current.uuid(), + current.isExternal()); tableMetadataStore.put(identifier.getFullName(), newTableMetadata); - return; + } catch (Catalog.TableNotExistException + | Catalog.ColumnAlreadyExistException + | Catalog.ColumnNotExistException + | RuntimeException e) { + throw e; + } catch (Exception e) { + throw new RuntimeException(e); } - catalog.alterTable(identifier, changes, false); - FileStoreTable table = (FileStoreTable) catalog.getTable(identifier); - TableSchema newSchema = table.schema(); - TableMetadata newTableMetadata = - createTableMetadata( - identifier, - newSchema.id(), - newSchema.toSchema(), - tableMetadata.uuid(), - tableMetadata.isExternal()); - tableMetadataStore.put(identifier.getFullName(), newTableMetadata); - } catch (Catalog.TableNotExistException - | Catalog.ColumnAlreadyExistException - | Catalog.ColumnNotExistException - | RuntimeException e) { - throw e; - } catch (Exception e) { - throw new RuntimeException(e); } } } @@ -3113,6 +3309,574 @@ private static int getMaxResults(Map parameters) { return maxResults; } + private MockResponse permissionsApiHandler( + String method, String resourcePath, String data, Map parameters) + throws JsonProcessingException { + if ("GET".equals(method) && permissionUri.equals(resourcePath)) { + PermissionResource target = permissionResource(parameters); + MockResponse authorization = validateManagementPermission(target); + if (authorization != null) { + return authorization; + } + MockResponse validation = validateResourceAndPrincipal(target, parameters); + if (validation != null) { + return validation; + } + List filtered = permissionStore.list(target, parameters); + int start = + parameters.containsKey(PAGE_TOKEN) + ? Integer.parseInt(parameters.get(PAGE_TOKEN)) + : 0; + int end = Math.min(start + getPermissionMaxResults(parameters), filtered.size()); + List page = new ArrayList<>(filtered.subList(start, end)); + String nextPageToken = end < filtered.size() ? String.valueOf(end) : null; + return mockResponse(new ListPermissionsResponse(page, nextPageToken), 200); + } + + if ("POST".equals(method) && (permissionUri + "/grant").equals(resourcePath)) { + PermissionAssignment assignment = + RESTApi.fromJson(data, GrantPermissionRequest.class).assignment(); + MockResponse authorization = validateManagementPermission(assignment.getResource()); + if (authorization != null) { + return authorization; + } + MockResponse validation = + validateResourceAndPrincipal( + assignment.getResource(), assignment.getPrincipal()); + if (validation != null) { + return validation; + } + validation = validateColumnAssignment(assignment); + if (validation != null) { + return validation; + } + permissionStore.put(assignment); + return new MockResponse().setResponseCode(200); + } + + if ("POST".equals(method) && (permissionUri + "/revoke").equals(resourcePath)) { + RevokePermissionRequest request = RESTApi.fromJson(data, RevokePermissionRequest.class); + MockResponse authorization = validateManagementPermission(request.getResource()); + if (authorization != null) { + return authorization; + } + MockResponse validation = + validateResourceAndPrincipal(request.getResource(), request.getPrincipal()); + if (validation != null) { + return validation; + } + permissionStore.remove( + request.getResource(), request.getAccess(), request.getPrincipal()); + return new MockResponse().setResponseCode(200); + } + + return new MockResponse().setResponseCode(404); + } + + @Nullable + private MockResponse validateResourceAndPrincipal( + PermissionResource resource, Map parameters) { + MockResponse resourceError = validateResource(resource); + if (resourceError != null || !parameters.containsKey("principal")) { + return resourceError; + } + return validatePrincipal(parameters.get("principal")); + } + + @Nullable + private MockResponse validateResourceAndPrincipal( + PermissionResource resource, String principal) { + MockResponse resourceError = validateResource(resource); + return resourceError == null ? validatePrincipal(principal) : resourceError; + } + + @Nullable + private MockResponse validateColumnAssignment(PermissionAssignment assignment) { + if (assignment.getResource().getType() != ResourceType.COLUMN) { + return null; + } + PermissionResource resource = assignment.getResource(); + Identifier identifier = Identifier.create(resource.getDatabase(), resource.getTable()); + TableMetadata metadata = tableMetadataStore.get(identifier.getFullName()); + RESTColumnPermissionSupport.ValidationError error = + RESTColumnPermissionSupport.validate(assignment, metadata); + return error == null + ? null + : mockResponse( + new ErrorResponse( + error.resourceType, error.resourceName, error.message, error.code), + error.code); + } + + @Nullable + private MockResponse validateResource(PermissionResource resource) { + boolean exists; + switch (resource.getType()) { + case CATALOG: + case CATALOG_ALL: + exists = true; + break; + case DATABASE: + case DATABASE_ALL: + exists = databaseStore.containsKey(resource.getDatabase()); + break; + case TABLE: + case COLUMN: + exists = + tableMetadataStore.containsKey( + Identifier.create(resource.getDatabase(), resource.getTable()) + .getFullName()); + break; + case FUNCTION: + exists = + functionStore.containsKey( + Identifier.create(resource.getDatabase(), resource.getFunction()) + .getFullName()); + break; + case VIEW: + exists = + viewStore.containsKey( + Identifier.create(resource.getDatabase(), resource.getView()) + .getFullName()); + break; + default: + exists = false; + } + return exists + ? null + : mockResponse( + new ErrorResponse( + resource.getType().name(), + resourceName(resource), + "Permission resource does not exist.", + 404), + 404); + } + + @Nullable + private MockResponse validateManagementPermission(PermissionResource resource) { + return noManagementPermissionResources.contains(resource) + ? mockResponse( + new ErrorResponse( + resource.getType().name(), + resourceName(resource), + "The caller cannot manage permissions on this resource.", + 403), + 403) + : null; + } + + @Nullable + private MockResponse validatePrincipal(String principal) { + return managementPrincipals.contains(principal) + ? null + : mockResponse( + new ErrorResponse( + "PRINCIPAL", + principal, + "Permission principal does not exist.", + 404), + 404); + } + + private static String resourceName(PermissionResource resource) { + switch (resource.getType()) { + case CATALOG: + case CATALOG_ALL: + return "catalog"; + case DATABASE: + case DATABASE_ALL: + return resource.getDatabase(); + case TABLE: + case COLUMN: + return resource.getDatabase() + "." + resource.getTable(); + case FUNCTION: + return resource.getDatabase() + "." + resource.getFunction(); + case VIEW: + return resource.getDatabase() + "." + resource.getView(); + default: + return resource.getType().name(); + } + } + + private static int getPermissionMaxResults(Map parameters) { + String strMaxResults = parameters.get(MAX_RESULTS); + if (strMaxResults == null) { + return DEFAULT_MAX_RESULTS; + } + int maxResults = Integer.parseInt(strMaxResults); + return Math.max(1, Math.min(maxResults, ListPermissionsRequest.MAX_PAGE_SIZE)); + } + + private static boolean matches( + Map parameters, String key, @Nullable String value) { + return !parameters.containsKey(key) || Objects.equals(parameters.get(key), value); + } + + private static PermissionResource permissionResource(Map parameters) { + return new PermissionResource( + ResourceType.fromString(parameters.get("resourceType")), + parameters.get("database"), + parameters.get("table"), + parameters.get("function"), + parameters.get("view")); + } + + private boolean isPolicyPath(String resourcePath) { + try { + policyPath(resourcePath); + return true; + } catch (IllegalArgumentException e) { + return false; + } + } + + private MockResponse policiesApiHandler( + String method, String resourcePath, String data, Map parameters) + throws JsonProcessingException { + PolicyPath path = policyPath(resourcePath); + MockResponse authorization = validateManagementPermission(path.resource); + if (authorization != null) { + return authorization; + } + MockResponse resourceError = validateResource(path.resource); + if (resourceError != null) { + return resourceError; + } + String tableUuid = tableUuid(path.resource); + if ("GET".equals(method)) { + if (parameters.containsKey("principal")) { + MockResponse principalError = validatePrincipal(parameters.get("principal")); + if (principalError != null) { + return principalError; + } + } + List filtered = + policyStore.entrySet().stream() + .filter(entry -> entry.getKey().tableUuid.equals(tableUuid)) + .map(entry -> withResource(entry.getValue(), path.resource)) + .filter(policy -> matchesPolicy(policy, parameters)) + .sorted( + Comparator.comparing( + policy -> new PolicyKey(tableUuid, policy))) + .collect(Collectors.toList()); + int start = + parameters.containsKey(PAGE_TOKEN) + ? Integer.parseInt(parameters.get(PAGE_TOKEN)) + : 0; + int end = Math.min(start + getPermissionMaxResults(parameters), filtered.size()); + String nextPageToken = end < filtered.size() ? String.valueOf(end) : null; + return mockResponse( + new ListPoliciesResponse( + new ArrayList<>(filtered.subList(start, end)), nextPageToken), + 200); + } + + if ("POST".equals(method) && !path.drop) { + DataPolicy policy = RESTApi.fromJson(data, PolicyRequest.class).policy(path.resource); + String resourceName = policyResourceName(policy); + synchronized (policyLock(tableUuid)) { + MockResponse targetError = validatePolicyTableVersion(path.resource, tableUuid); + if (targetError != null) { + return targetError; + } + MockResponse validation = validatePolicy(policy); + if (validation != null) { + return validation; + } + policy = canonicalizePolicy(policy); + PolicyKey key = new PolicyKey(tableUuid, policy); + if (policyStore.putIfAbsent(key, policy) != null) { + return mockResponse( + new ErrorResponse( + ErrorResponse.RESOURCE_TYPE_POLICY, + resourceName, + "Policy already exists.", + 409), + 409); + } + } + return new MockResponse().setResponseCode(200); + } + + if ("POST".equals(method) && path.drop) { + DropPolicyRequest request = RESTApi.fromJson(data, DropPolicyRequest.class); + DataPolicy existing; + synchronized (policyLock(tableUuid)) { + MockResponse targetError = validatePolicyTableVersion(path.resource, tableUuid); + if (targetError != null) { + return targetError; + } + existing = + policyStore.remove( + new PolicyKey( + tableUuid, + request.getType(), + request.getPrincipal(), + request.getColumn())); + } + if (existing == null) { + return mockResponse( + new ErrorResponse( + ErrorResponse.RESOURCE_TYPE_POLICY, + policyResourceName(request), + "Policy does not exist.", + 404), + 404); + } + return new MockResponse().setResponseCode(200); + } + + return new MockResponse().setResponseCode(404); + } + + @Nullable + private MockResponse validatePolicy(DataPolicy policy) { + Identifier identifier = + Identifier.create( + policy.getResource().getDatabase(), policy.getResource().getTable()); + TableMetadata metadata = tableMetadataStore.get(identifier.getFullName()); + if (!CoreOptions.fromMap(metadata.schema().options()).queryAuthEnabled()) { + return mockResponse( + new ErrorResponse( + ErrorResponse.RESOURCE_TYPE_TABLE, + identifier.getFullName(), + "Data policies require the target table option query-auth.enabled=true.", + 409), + 409); + } + MockResponse principalError = validatePrincipal(policy.getPrincipal()); + if (principalError != null) { + return principalError; + } + + RowFilter rowFilter = policy.getRowFilter(); + try { + Set columns = new HashSet<>(metadata.schema().fieldNames()); + ColumnMask columnMask = policy.getColumnMask(); + if (columnMask != null) { + checkArgument( + columns.contains(columnMask.getOnColumn()), + "Policy column %s does not exist in table %s.", + columnMask.getOnColumn(), + identifier.getFullName()); + } + if (rowFilter == null) { + parseColumnMask(metadata.schema(), columnMask); + } else { + parseRowFilter(metadata.schema(), rowFilter); + } + } catch (RuntimeException e) { + return mockResponse( + new ErrorResponse( + ErrorResponse.RESOURCE_TYPE_POLICY, + policyResourceName(policy), + e.getMessage(), + 400), + 400); + } + return null; + } + + private static String policyResourceName(DataPolicy policy) { + ColumnMask columnMask = policy.getColumnMask(); + return policy.type().name() + + ":" + + policy.getPrincipal() + + (columnMask == null ? "" : ":" + columnMask.getOnColumn()); + } + + private static String policyResourceName(DropPolicyRequest request) { + return request.getType().name() + + ":" + + request.getPrincipal() + + (request.getColumn() == null ? "" : ":" + request.getColumn()); + } + + private String tableUuid(PermissionResource resource) { + return tableMetadataStore + .get(Identifier.create(resource.getDatabase(), resource.getTable()).getFullName()) + .uuid(); + } + + private Object policyLock(String tableUuid) { + return tablePolicyLocks.computeIfAbsent(tableUuid, ignored -> new Object()); + } + + @Nullable + private MockResponse validatePolicyTableVersion( + PermissionResource resource, String expectedTableUuid) { + MockResponse resourceError = validateResource(resource); + if (resourceError != null) { + return resourceError; + } + String currentTableUuid = tableUuid(resource); + return expectedTableUuid.equals(currentTableUuid) + ? null + : mockResponse( + new ErrorResponse( + ErrorResponse.RESOURCE_TYPE_TABLE, + resourceName(resource), + "Table changed while managing its policies.", + 409), + 409); + } + + private static DataPolicy withResource(DataPolicy policy, PermissionResource resource) { + return policy.getRowFilter() == null + ? DataPolicy.columnMask(resource, policy.getColumnMask(), policy.getPrincipal()) + : DataPolicy.rowFilter(resource, policy.getRowFilter(), policy.getPrincipal()); + } + + private DataPolicy canonicalizePolicy(DataPolicy policy) { + Identifier identifier = + Identifier.create( + policy.getResource().getDatabase(), policy.getResource().getTable()); + TableSchema schema = tableMetadataStore.get(identifier.getFullName()).schema(); + if (policy.getRowFilter() != null) { + String predicate = + JsonSerdeUtil.toFlatJson(parseRowFilter(schema, policy.getRowFilter())); + return DataPolicy.rowFilter( + policy.getResource(), new RowFilter(predicate), policy.getPrincipal()); + } + ColumnMask columnMask = policy.getColumnMask(); + String transform = JsonSerdeUtil.toFlatJson(parseColumnMask(schema, columnMask)); + return DataPolicy.columnMask( + policy.getResource(), + new ColumnMask(columnMask.getOnColumn(), transform), + policy.getPrincipal()); + } + + private static Predicate parseRowFilter(TableSchema schema, RowFilter rowFilter) { + Predicate predicate = JsonSerdeUtil.fromJson(rowFilter.getPredicate(), Predicate.class); + checkArgument(predicate != null, "Row filter predicate cannot be JSON null."); + Predicate remapped = + TableQueryAuthResult.remapPredicate(predicate, schema.logicalRowType()); + checkArgument(remapped != null, "Row filter predicate cannot be empty."); + return remapped; + } + + private static Transform parseColumnMask(TableSchema schema, ColumnMask columnMask) { + Transform transform = JsonSerdeUtil.fromJson(columnMask.getTransform(), Transform.class); + checkArgument(transform != null, "Column mask transform cannot be JSON null."); + RowType rowType = schema.logicalRowType(); + List remappedInputs = new ArrayList<>(); + for (Object input : transform.inputs()) { + if (input instanceof FieldRef) { + FieldRef ref = (FieldRef) input; + int index = rowType.getFieldIndex(ref.name()); + checkArgument( + index >= 0, + "Column masking refers to field '%s' which is not present in table schema.", + ref.name()); + remappedInputs.add(new FieldRef(index, ref.name(), rowType.getTypeAt(index))); + } else { + remappedInputs.add(input); + } + } + Transform remapped = transform.copyWithNewInputs(remappedInputs); + int targetIndex = rowType.getFieldIndex(columnMask.getOnColumn()); + checkArgument( + targetIndex >= 0, + "Policy column %s does not exist in table schema.", + columnMask.getOnColumn()); + checkArgument( + rowType.getTypeAt(targetIndex).equals(remapped.outputType()), + "Column mask output type %s does not match target column %s type %s.", + remapped.outputType(), + columnMask.getOnColumn(), + rowType.getTypeAt(targetIndex)); + return remapped; + } + + private void removePolicies(@Nullable String tableUuid) { + if (tableUuid != null) { + policyStore.keySet().removeIf(key -> key.tableUuid.equals(tableUuid)); + } + } + + private void validatePoliciesForSchema( + Identifier identifier, @Nullable String tableUuid, TableSchema schema) { + if (tableUuid == null) { + return; + } + List policies = + policyStore.entrySet().stream() + .filter(entry -> entry.getKey().tableUuid.equals(tableUuid)) + .map(Map.Entry::getValue) + .collect(Collectors.toList()); + if (policies.isEmpty()) { + return; + } + checkArgument( + CoreOptions.fromMap(schema.options()).queryAuthEnabled(), + "Cannot disable query-auth.enabled while table %s has data policies.", + identifier.getFullName()); + + Set columns = new HashSet<>(schema.fieldNames()); + for (DataPolicy policy : policies) { + ColumnMask columnMask = policy.getColumnMask(); + if (columnMask != null) { + checkArgument( + columns.contains(columnMask.getOnColumn()), + "Cannot remove or rename policy column %s from table %s.", + columnMask.getOnColumn(), + identifier.getFullName()); + } + if (policy.getRowFilter() == null) { + parseColumnMask(schema, columnMask); + } else { + parseRowFilter(schema, policy.getRowFilter()); + } + } + } + + private static boolean matchesPolicy(DataPolicy policy, Map parameters) { + if (!matches(parameters, "type", policy.type().name())) { + return false; + } + if (parameters.containsKey("column")) { + ColumnMask columnMask = policy.getColumnMask(); + if (columnMask == null + || !Objects.equals(parameters.get("column"), columnMask.getOnColumn())) { + return false; + } + } + return !parameters.containsKey("principal") + || policy.getPrincipal().equals(parameters.get("principal")); + } + + private PolicyPath policyPath(String resourcePath) { + String catalogBase = StringUtils.substringBeforeLast(permissionUri, "/"); + checkArgument(resourcePath.startsWith(catalogBase + "/"), "Not a catalog policy path."); + String[] parts = resourcePath.substring(catalogBase.length() + 1).split("/"); + if ((parts.length == 5 || (parts.length == 6 && "drop".equals(parts[5]))) + && "databases".equals(parts[0]) + && "tables".equals(parts[2]) + && "policies".equals(parts[4])) { + return new PolicyPath( + new PermissionResource( + ResourceType.TABLE, + RESTUtil.decodeString(parts[1]), + RESTUtil.decodeString(parts[3]), + null, + null), + parts.length == 6); + } + throw new IllegalArgumentException("Not a policy path."); + } + + private static class PolicyPath { + + private final PermissionResource resource; + private final boolean drop; + + private PolicyPath(PermissionResource resource, boolean drop) { + this.resource = resource; + this.drop = drop; + } + } + private String getNextPageTokenForEntities(List entities, Integer maxResults) { if (entities == null || entities.isEmpty() diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/RESTColumnPermissionSupport.java b/paimon-core/src/test/java/org/apache/paimon/rest/RESTColumnPermissionSupport.java new file mode 100644 index 000000000000..39f7d72d3305 --- /dev/null +++ b/paimon-core/src/test/java/org/apache/paimon/rest/RESTColumnPermissionSupport.java @@ -0,0 +1,135 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.rest; + +import org.apache.paimon.CoreOptions; +import org.apache.paimon.catalog.Identifier; +import org.apache.paimon.catalog.TableMetadata; +import org.apache.paimon.management.PermissionAssignment; +import org.apache.paimon.management.PermissionColumns; +import org.apache.paimon.management.PermissionResource; +import org.apache.paimon.management.ResourceType; +import org.apache.paimon.rest.responses.ErrorResponse; + +import javax.annotation.Nullable; + +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; + +/** Column permission validation and composition used by the REST catalog test server. */ +final class RESTColumnPermissionSupport { + + private RESTColumnPermissionSupport() {} + + static boolean canSelect( + RESTPermissionStore store, + Set principals, + Identifier identifier, + TableMetadata metadata, + @Nullable List selectedColumns) { + PermissionResource resource = + new PermissionResource( + ResourceType.COLUMN, + identifier.getDatabaseName(), + identifier.getTableName(), + null, + null); + List assignments = + store.list(resource, Collections.emptyMap()).stream() + .filter(assignment -> principals.contains(assignment.getPrincipal())) + .filter(RESTColumnPermissionSupport::notExpired) + .collect(Collectors.toList()); + if (assignments.isEmpty()) { + return true; + } + + Set included = new HashSet<>(metadata.schema().fieldNames()); + for (PermissionAssignment assignment : assignments) { + PermissionColumns columns = assignment.getColumns(); + if (columns.getColumnNames() != null) { + included.retainAll(columns.getColumnNames()); + } else { + included.removeAll(columns.getExcludedColumnNames()); + } + } + List selected = + selectedColumns == null ? metadata.schema().fieldNames() : selectedColumns; + for (String column : selected) { + int nestedSeparator = column.indexOf('.'); + String topLevel = nestedSeparator < 0 ? column : column.substring(0, nestedSeparator); + if (!included.contains(topLevel)) { + return false; + } + } + return true; + } + + @Nullable + static ValidationError validate(PermissionAssignment assignment, TableMetadata metadata) { + if (!CoreOptions.fromMap(metadata.schema().options()).queryAuthEnabled()) { + return new ValidationError( + ErrorResponse.RESOURCE_TYPE_TABLE, + assignment.getResource().getDatabase() + + "." + + assignment.getResource().getTable(), + "Column permissions require query-auth.enabled=true.", + 409); + } + Set tableColumns = new HashSet<>(metadata.schema().fieldNames()); + PermissionColumns columns = assignment.getColumns(); + List referenced = + columns.getColumnNames() == null + ? columns.getExcludedColumnNames() + : columns.getColumnNames(); + for (String column : referenced) { + if (!tableColumns.contains(column)) { + return new ValidationError( + ErrorResponse.RESOURCE_TYPE_COLUMN, + column, + "Permission column does not exist.", + 404); + } + } + return null; + } + + private static boolean notExpired(PermissionAssignment assignment) { + return assignment.getExpireTime() == null + || java.time.Instant.now() + .isBefore(java.time.Instant.parse(assignment.getExpireTime())); + } + + static final class ValidationError { + final String resourceType; + final String resourceName; + final String message; + final int code; + + private ValidationError( + String resourceType, String resourceName, String message, int code) { + this.resourceType = resourceType; + this.resourceName = resourceName; + this.message = message; + this.code = code; + } + } +} diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/RESTPermissionStore.java b/paimon-core/src/test/java/org/apache/paimon/rest/RESTPermissionStore.java new file mode 100644 index 000000000000..83d28bc6010b --- /dev/null +++ b/paimon-core/src/test/java/org/apache/paimon/rest/RESTPermissionStore.java @@ -0,0 +1,114 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.rest; + +import org.apache.paimon.management.PermissionAssignment; +import org.apache.paimon.management.PermissionResource; + +import java.util.Comparator; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.concurrent.ConcurrentHashMap; +import java.util.stream.Collectors; + +/** Atomic permission assignment store for the REST catalog test server. */ +final class RESTPermissionStore { + + private final Map assignments = new ConcurrentHashMap<>(); + + void put(PermissionAssignment assignment) { + assignments.put(PermissionKey.fromAssignment(assignment), assignment); + } + + void remove(PermissionResource resource, String access, String principal) { + assignments.remove(new PermissionKey(resource, access, principal)); + } + + List list(PermissionResource target, Map parameters) { + return assignments.values().stream() + .filter(assignment -> assignment.getResource().equals(target)) + .filter(assignment -> matches(parameters, "principal", assignment.getPrincipal())) + .filter(assignment -> matches(parameters, "access", assignment.getAccess())) + .sorted(Comparator.comparing(RESTPermissionStore::sortKey)) + .collect(Collectors.toList()); + } + + private static boolean matches(Map parameters, String key, String value) { + return !parameters.containsKey(key) || parameters.get(key).equals(value); + } + + private static String sortKey(PermissionAssignment assignment) { + PermissionResource source = assignment.getResource(); + return source.getType().name() + + '\0' + + value(source.getDatabase()) + + '\0' + + value(source.getTable()) + + '\0' + + value(source.getFunction()) + + '\0' + + value(source.getView()) + + '\0' + + assignment.getAccess() + + '\0' + + assignment.getPrincipal(); + } + + private static String value(String value) { + return value == null ? "" : value; + } + + private static class PermissionKey { + + private final PermissionResource resource; + private final String access; + private final String principal; + + private PermissionKey(PermissionResource resource, String access, String principal) { + this.resource = resource; + this.access = access; + this.principal = principal; + } + + private static PermissionKey fromAssignment(PermissionAssignment assignment) { + return new PermissionKey( + assignment.getResource(), assignment.getAccess(), assignment.getPrincipal()); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof PermissionKey)) { + return false; + } + PermissionKey that = (PermissionKey) o; + return resource.equals(that.resource) + && access.equals(that.access) + && principal.equals(that.principal); + } + + @Override + public int hashCode() { + return Objects.hash(resource, access, principal); + } + } +} diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/RESTPermissionStoreTest.java b/paimon-core/src/test/java/org/apache/paimon/rest/RESTPermissionStoreTest.java new file mode 100644 index 000000000000..73dfacb12976 --- /dev/null +++ b/paimon-core/src/test/java/org/apache/paimon/rest/RESTPermissionStoreTest.java @@ -0,0 +1,123 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.rest; + +import org.apache.paimon.management.PermissionAssignment; +import org.apache.paimon.management.PermissionColumns; +import org.apache.paimon.management.PermissionResource; +import org.apache.paimon.management.ResourceType; + +import org.junit.jupiter.api.Test; + +import java.time.Instant; +import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; +import java.util.stream.IntStream; + +import static org.assertj.core.api.Assertions.assertThat; + +/** Tests atomic replacement and exact-resource filtering in {@link RESTPermissionStore}. */ +class RESTPermissionStoreTest { + + private static final String ANALYST = "analyst"; + + @Test + void testConcurrentGrantReplacesTheSameIdentity() { + RESTPermissionStore store = new RESTPermissionStore(); + PermissionResource table = tableResource(); + + IntStream.range(0, 1000) + .parallel() + .forEach( + i -> + store.put( + new PermissionAssignment( + table, + "SELECT", + ANALYST, + Instant.ofEpochSecond(i).toString()))); + + assertThat(store.list(table, tableParameters())).hasSize(1); + } + + @Test + void testListReturnsOnlyTheExactTarget() { + RESTPermissionStore store = new RESTPermissionStore(); + PermissionResource catalog = + new PermissionResource(ResourceType.CATALOG, null, null, null, null); + store.put(new PermissionAssignment(catalog, "CREATEDATABASE", ANALYST, null)); + store.put(new PermissionAssignment(tableResource(), "SELECT", ANALYST, null)); + + assertThat(store.list(tableResource(), tableParameters())) + .singleElement() + .extracting(PermissionAssignment::getAccess) + .isEqualTo("SELECT"); + } + + @Test + void testColumnGrantReplacesTheWholeColumnRangeForTheSameIdentity() { + RESTPermissionStore store = new RESTPermissionStore(); + PermissionResource column = columnResource(); + store.put( + new PermissionAssignment( + column, + "SELECT", + ANALYST, + new PermissionColumns(Arrays.asList("id", "region"), null), + null)); + store.put( + new PermissionAssignment( + column, + "SELECT", + ANALYST, + new PermissionColumns(null, Arrays.asList("email")), + null)); + + assertThat(store.list(column, columnParameters())) + .singleElement() + .extracting(PermissionAssignment::getColumns) + .extracting(PermissionColumns::getExcludedColumnNames) + .isEqualTo(Arrays.asList("email")); + } + + private static PermissionResource tableResource() { + return new PermissionResource(ResourceType.TABLE, "sales", "orders", null, null); + } + + private static PermissionResource columnResource() { + return new PermissionResource(ResourceType.COLUMN, "sales", "orders", null, null); + } + + private static Map tableParameters() { + Map parameters = new HashMap<>(); + parameters.put("resourceType", "TABLE"); + parameters.put("database", "sales"); + parameters.put("table", "orders"); + return parameters; + } + + private static Map columnParameters() { + Map parameters = new HashMap<>(); + parameters.put("resourceType", "COLUMN"); + parameters.put("database", "sales"); + parameters.put("table", "orders"); + return parameters; + } +} diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/TableLifecycleLocks.java b/paimon-core/src/test/java/org/apache/paimon/rest/TableLifecycleLocks.java new file mode 100644 index 000000000000..450e9883ff3b --- /dev/null +++ b/paimon-core/src/test/java/org/apache/paimon/rest/TableLifecycleLocks.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.rest; + +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +/** Stable, deadlock-ordered locks for table-name lifecycle operations in the test server. */ +final class TableLifecycleLocks { + + private final Map locks = new ConcurrentHashMap<>(); + + Object lock(String tableName) { + return locks.computeIfAbsent(tableName, ignored -> new Object()); + } + + Object[] ordered(String left, String right) { + return left.compareTo(right) <= 0 + ? new Object[] {lock(left), lock(right)} + : new Object[] {lock(right), lock(left)}; + } +} From 86e57711795ea1dfa4a2a55f82277363da817c5e Mon Sep 17 00:00:00 2001 From: JingsongLi Date: Wed, 26 Aug 2026 18:48:07 +0800 Subject: [PATCH 3/5] [fix] Harden REST management lifecycle handling --- docs/docs/concepts/rest/management-api.md | 9 +- docs/static/rest-management-open-api.yaml | 4 +- .../paimon/catalog/TableQueryAuthResult.java | 15 +- .../table/source/AbstractDataTableRead.java | 29 +- .../org/apache/paimon/rest/PolicyKey.java | 11 + .../org/apache/paimon/rest/PolicyKeyTest.java | 18 ++ .../apache/paimon/rest/RESTCatalogServer.java | 283 +++++++++++++----- .../paimon/rest/RESTPermissionStore.java | 222 ++++++++++++-- .../paimon/rest/RESTPermissionStoreTest.java | 190 ++++++++++++ .../source/AbstractDataTableReadTest.java | 136 +++++++++ 10 files changed, 795 insertions(+), 122 deletions(-) create mode 100644 paimon-core/src/test/java/org/apache/paimon/table/source/AbstractDataTableReadTest.java diff --git a/docs/docs/concepts/rest/management-api.md b/docs/docs/concepts/rest/management-api.md index 95892fa8a211..751eb6f3afa6 100644 --- a/docs/docs/concepts/rest/management-api.md +++ b/docs/docs/concepts/rest/management-api.md @@ -170,10 +170,11 @@ are intersected. If any applicable range rejects a selected column, the query fa silently dropping that column. Schema evolution keeps the assignment attached to the stable table identity. Renaming a referenced -column updates its stored name. Dropping a referenced column removes it from the range; if that -would leave the stored list empty, the assignment is removed. An allowlist denies columns added -later, while a denylist allows them, so allowlists are safer when new columns may contain sensitive -data. +column updates its stored name. Dropping a referenced column removes it from the range. The server +must reject a schema change that would leave an allowlist empty because removing that assignment +would widen access; an empty denylist is equivalent to no column restriction, so that assignment is +removed. An allowlist denies columns added later, while a denylist allows them, so allowlists are +safer when new columns may contain sensitive data. `expireTime`, when present, is an exclusive upper bound evaluated against the REST server clock. At `now >= expireTime`, the assignment must not authorize access. Expired direct assignments may diff --git a/docs/static/rest-management-open-api.yaml b/docs/static/rest-management-open-api.yaml index 4da5a572b5b5..8ad9ed16c96f 100644 --- a/docs/static/rest-management-open-api.yaml +++ b/docs/static/rest-management-open-api.yaml @@ -638,7 +638,9 @@ components: names must exist when granted. An allowlist denies columns added later, while a denylist allows columns added later. All applicable column ranges are intersected, and selecting any column outside the effective range fails the query. The target table must enforce query - authorization before the grant becomes visible. + authorization before the grant becomes visible. Schema evolution must reject removal of + every column in an allowlist because deleting the resulting empty assignment would widen + access; an empty denylist assignment may be removed. properties: columnNames: type: array diff --git a/paimon-core/src/main/java/org/apache/paimon/catalog/TableQueryAuthResult.java b/paimon-core/src/main/java/org/apache/paimon/catalog/TableQueryAuthResult.java index 6d381078c739..a2a113a5e897 100644 --- a/paimon-core/src/main/java/org/apache/paimon/catalog/TableQueryAuthResult.java +++ b/paimon-core/src/main/java/org/apache/paimon/catalog/TableQueryAuthResult.java @@ -133,7 +133,15 @@ public Map extractColumnMasking() { public RecordReader doAuth( RecordReader reader, RowType outputRowType) { - Predicate rowFilter = extractPredicate(); + return doAuth(reader, outputRowType, extractPredicate(), extractColumnMasking()); + } + + /** Applies already decoded query-authorization definitions to a physical read projection. */ + public RecordReader doAuth( + RecordReader reader, + RowType outputRowType, + @Nullable Predicate rowFilter, + Map selectedColumnMasking) { if (rowFilter != null) { Predicate remappedFilter = remapPredicate(rowFilter, outputRowType); if (remappedFilter != null) { @@ -141,10 +149,9 @@ public RecordReader doAuth( } } - Map columnMasking = extractColumnMasking(); - if (columnMasking != null && !columnMasking.isEmpty()) { + if (!selectedColumnMasking.isEmpty()) { Map remappedMasking = - transformRemapping(outputRowType, columnMasking); + transformRemapping(outputRowType, selectedColumnMasking); if (!remappedMasking.isEmpty()) { reader = reader.transform(row -> transform(outputRowType, remappedMasking, row)); } diff --git a/paimon-core/src/main/java/org/apache/paimon/table/source/AbstractDataTableRead.java b/paimon-core/src/main/java/org/apache/paimon/table/source/AbstractDataTableRead.java index 6d6f572d2b12..9fbb5827813e 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/source/AbstractDataTableRead.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/source/AbstractDataTableRead.java @@ -27,14 +27,15 @@ import org.apache.paimon.predicate.Transform; import org.apache.paimon.reader.RecordReader; import org.apache.paimon.schema.TableSchema; +import org.apache.paimon.types.DataField; import org.apache.paimon.types.RowType; -import org.apache.paimon.utils.ListUtils; import org.apache.paimon.utils.ProjectedRow; import javax.annotation.Nullable; import java.io.IOException; import java.util.ArrayList; +import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; @@ -143,11 +144,19 @@ private RecordReader authedReader(Split split, TableQueryAuthResult Predicate authPredicate = authResult.extractPredicate(); Map columnMasking = authResult.extractColumnMasking(); ProjectedRow backRow = null; + List readFields = readType.getFieldNames(); + Set readFieldSet = new HashSet<>(readFields); + Map selectedColumnMasking = new HashMap<>(); + for (Map.Entry mask : columnMasking.entrySet()) { + if (readFieldSet.contains(mask.getKey())) { + selectedColumnMasking.put(mask.getKey(), mask.getValue()); + } + } Set authFields = new HashSet<>(); if (authPredicate != null) { authFields.addAll(collectFieldNames(authPredicate)); } - for (Map.Entry mask : columnMasking.entrySet()) { + for (Map.Entry mask : selectedColumnMasking.entrySet()) { authFields.add(mask.getKey()); for (Object input : mask.getValue().inputs()) { if (input instanceof FieldRef) { @@ -156,21 +165,19 @@ private RecordReader authedReader(Split split, TableQueryAuthResult } } if (!authFields.isEmpty()) { - List readFields = readType.getFieldNames(); - List authAddNames = new ArrayList<>(); - Set readFieldSet = new HashSet<>(readFields); - for (String field : tableType.getFieldNames()) { - if (authFields.contains(field) && !readFieldSet.contains(field)) { - authAddNames.add(field); + List expandedFields = new ArrayList<>(readType.getFields()); + for (DataField field : tableType.getFields()) { + if (authFields.contains(field.name()) && !readFieldSet.contains(field.name())) { + expandedFields.add(field); } } - if (!authAddNames.isEmpty()) { - readType = tableType.project(ListUtils.union(readFields, authAddNames)); + if (expandedFields.size() > readType.getFieldCount()) { + readType = readType.copy(expandedFields); applyReadType(readType); backRow = ProjectedRow.from(readType.projectIndexes(readFields)); } } - reader = authResult.doAuth(reader(split), readType); + reader = authResult.doAuth(reader(split), readType, authPredicate, selectedColumnMasking); if (backRow != null) { reader = reader.transform(backRow::replaceRow); } diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/PolicyKey.java b/paimon-core/src/test/java/org/apache/paimon/rest/PolicyKey.java index 325c1bceb2ad..44a86d46510e 100644 --- a/paimon-core/src/test/java/org/apache/paimon/rest/PolicyKey.java +++ b/paimon-core/src/test/java/org/apache/paimon/rest/PolicyKey.java @@ -87,4 +87,15 @@ public int compareTo(PolicyKey that) { } return that.column == null ? 1 : column.compareTo(that.column); } + + String sortKey() { + return cursorPart(tableUuid) + + cursorPart(type.name()) + + cursorPart(principal) + + cursorPart(column); + } + + private static String cursorPart(String value) { + return value == null ? "-1:" : value.length() + ":" + value; + } } diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/PolicyKeyTest.java b/paimon-core/src/test/java/org/apache/paimon/rest/PolicyKeyTest.java index 027736ded674..7d659bf0d297 100644 --- a/paimon-core/src/test/java/org/apache/paimon/rest/PolicyKeyTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/rest/PolicyKeyTest.java @@ -53,4 +53,22 @@ void testOrderingDoesNotFlattenOpaquePrincipalAndColumn() { assertThat(sorted).containsExactly(second, first); } + + @Test + void testCursorKeyDoesNotFlattenOpaquePrincipalAndColumn() { + PermissionResource resource = + new PermissionResource(ResourceType.TABLE, "database", "table", null, null); + PolicyKey first = + new PolicyKey( + "table-id", + DataPolicy.columnMask( + resource, new ColumnMask("b\0\1c", "{\"name\":\"NULL\"}"), "a")); + PolicyKey second = + new PolicyKey( + "table-id", + DataPolicy.columnMask( + resource, new ColumnMask("c", "{\"name\":\"NULL\"}"), "a\0\1b")); + + assertThat(first.sortKey()).isNotEqualTo(second.sortKey()); + } } diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogServer.java b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogServer.java index b6e1270cb6e9..28d93c4f6ffb 100644 --- a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogServer.java +++ b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogServer.java @@ -158,10 +158,12 @@ import java.io.FileNotFoundException; import java.io.IOException; import java.io.UncheckedIOException; +import java.nio.charset.StandardCharsets; import java.time.Duration; import java.time.ZoneId; import java.util.ArrayList; import java.util.Arrays; +import java.util.Base64; import java.util.Collections; import java.util.Comparator; import java.util.HashMap; @@ -177,6 +179,7 @@ import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.function.Supplier; import java.util.regex.Pattern; import java.util.stream.Collectors; @@ -217,7 +220,7 @@ public class RESTCatalogServer { private final RESTFileSystemCatalog catalog; private final MockWebServer server; - private final Map databaseStore = new HashMap<>(); + private final Map databaseStore = new ConcurrentHashMap<>(); private final Map tableMetadataStore = new ConcurrentHashMap<>(); private final RESTPermissionStore permissionStore = new RESTPermissionStore(); private final Map policyStore = new ConcurrentHashMap<>(); @@ -235,13 +238,13 @@ public class RESTCatalogServer { new ConcurrentLinkedQueue<>(); private final Map> tablePartitionsStore = new HashMap<>(); - private final Map viewStore = new HashMap<>(); + private final Map viewStore = new ConcurrentHashMap<>(); private final Map tableLatestSnapshotStore = new HashMap<>(); private final Map tableWithSnapshotId2SnapshotStore = new HashMap<>(); private final List noPermissionDatabases = new ArrayList<>(); private final List noPermissionTables = new ArrayList<>(); private final List noPermissionViews = new ArrayList<>(); - private final Map functionStore = new HashMap<>(); + private final Map functionStore = new ConcurrentHashMap<>(); private final Map> columnAuthHandler = new HashMap<>(); private final Map> rowFilterAuthHandler = new HashMap<>(); private final Map> columnMaskingAuthHandler = new HashMap<>(); @@ -1318,7 +1321,12 @@ private MockResponse functionApiHandler( Function function = functionStore.get(identifier.getFullName()); switch (method) { case "DELETE": - functionStore.remove(identifier.getFullName()); + permissionStore.executeAtomically( + () -> { + functionStore.remove(identifier.getFullName()); + permissionStore.removeFunction(identifier); + return null; + }); break; case "GET": GetFunctionResponse response = toGetFunctionResponse(function); @@ -1611,7 +1619,12 @@ private MockResponse databaseHandle(String method, String data, String databaseN case "DELETE": catalog.dropDatabase(databaseName, false, true); removeDatabaseTableState(databaseName); - databaseStore.remove(databaseName); + permissionStore.executeAtomically( + () -> { + databaseStore.remove(databaseName); + permissionStore.removeDatabase(databaseName); + return null; + }); return new MockResponse().setResponseCode(200); case "POST": AlterDatabaseRequest requestBody = @@ -1977,6 +1990,7 @@ private MockResponse tableHandle(String method, String data, Identifier identifi } } removePolicies(current.uuid()); + permissionStore.removeTable(identifier); tableMetadataStore.remove(identifier.getFullName(), current); tableLatestSnapshotStore.remove(identifier.getFullName()); tablePartitionsStore.remove(identifier.getFullName()); @@ -2009,12 +2023,14 @@ private MockResponse replaceTableHandle(Identifier identifier, String data) thro current.isExternal()) .schema(); validatePoliciesForSchema(identifier, current.uuid(), replacementSchema); + validatePermissionsForSchema(identifier, current.schema(), replacementSchema); if (isFormatTable(current.schema().toSchema()) || isFormatTable(newSchema)) { throw new UnsupportedOperationException( "replaceTable does not support format tables."); } catalog.replaceTable(identifier, newSchema, false); TableSchema replacedSchema = catalog.loadTableSchema(identifier); + permissionStore.evolveTableColumns(identifier, current.schema(), replacedSchema); TableMetadata newTableMetadata = createTableMetadata( identifier, @@ -2072,6 +2088,7 @@ private MockResponse renameTableHandle(String data) throws Exception { current.isExternal()); tableMetadataStore.remove(fromTable.getFullName(), current); tableMetadataStore.put(toTable.getFullName(), renamedMetadata); + permissionStore.renameTable(fromTable, toTable); } } } @@ -2895,7 +2912,12 @@ private MockResponse viewHandle(String method, Identifier identifier, String req } throw new Catalog.ViewNotExistException(identifier); case "DELETE": - viewStore.remove(identifier.getFullName()); + permissionStore.executeAtomically( + () -> { + viewStore.remove(identifier.getFullName()); + permissionStore.removeView(identifier); + return null; + }); return new MockResponse().setResponseCode(200); case "POST": if (viewStore.containsKey(identifier.getFullName())) { @@ -2982,11 +3004,16 @@ private MockResponse renameViewHandle(String data) throws Exception { if (viewStore.containsKey(toView.getFullName())) { throw new Catalog.ViewAlreadyExistException(toView); } - if (viewStore.containsKey(fromView.getFullName())) { - View view = viewStore.get(fromView.getFullName()); - viewStore.remove(fromView.getFullName()); - viewStore.put(toView.getFullName(), view); - } + permissionStore.executeAtomically( + () -> { + if (viewStore.containsKey(fromView.getFullName())) { + View view = viewStore.get(fromView.getFullName()); + viewStore.remove(fromView.getFullName()); + viewStore.put(toView.getFullName(), view); + permissionStore.renameView(fromView, toView); + } + return null; + }); return new MockResponse().setResponseCode(200); } @@ -3009,6 +3036,7 @@ protected void alterTableImpl(Identifier identifier, List changes) new LazyField<>(() -> false), new LazyField<>(() -> identifier)); validatePoliciesForSchema(identifier, current.uuid(), candidateSchema); + validatePermissionsForSchema(identifier, current.schema(), candidateSchema); if (isFormatTable(schema.toSchema())) { TableMetadata newTableMetadata = createTableMetadata( @@ -3018,11 +3046,14 @@ protected void alterTableImpl(Identifier identifier, List changes) current.uuid(), current.isExternal()); tableMetadataStore.put(identifier.getFullName(), newTableMetadata); + permissionStore.evolveTableColumns( + identifier, current.schema(), candidateSchema); return; } catalog.alterTable(identifier, changes, false); FileStoreTable table = (FileStoreTable) catalog.getTable(identifier); TableSchema newSchema = table.schema(); + permissionStore.evolveTableColumns(identifier, current.schema(), newSchema); TableMetadata newTableMetadata = createTableMetadata( identifier, @@ -3323,14 +3354,14 @@ private MockResponse permissionsApiHandler( return validation; } List filtered = permissionStore.list(target, parameters); - int start = - parameters.containsKey(PAGE_TOKEN) - ? Integer.parseInt(parameters.get(PAGE_TOKEN)) - : 0; - int end = Math.min(start + getPermissionMaxResults(parameters), filtered.size()); - List page = new ArrayList<>(filtered.subList(start, end)); - String nextPageToken = end < filtered.size() ? String.valueOf(end) : null; - return mockResponse(new ListPermissionsResponse(page, nextPageToken), 200); + PagedList page = + buildManagementPage( + filtered, + getPermissionMaxResults(parameters), + parameters.get(PAGE_TOKEN), + RESTPermissionStore::sortKey); + return mockResponse( + new ListPermissionsResponse(page.getElements(), page.getNextPageToken()), 200); } if ("POST".equals(method) && (permissionUri + "/grant").equals(resourcePath)) { @@ -3340,18 +3371,22 @@ private MockResponse permissionsApiHandler( if (authorization != null) { return authorization; } - MockResponse validation = - validateResourceAndPrincipal( - assignment.getResource(), assignment.getPrincipal()); - if (validation != null) { - return validation; - } - validation = validateColumnAssignment(assignment); - if (validation != null) { - return validation; - } - permissionStore.put(assignment); - return new MockResponse().setResponseCode(200); + return mutatePermission( + assignment.getResource(), + () -> { + MockResponse validation = + validateResourceAndPrincipal( + assignment.getResource(), assignment.getPrincipal()); + if (validation != null) { + return validation; + } + validation = validateColumnAssignment(assignment); + if (validation != null) { + return validation; + } + permissionStore.put(assignment); + return new MockResponse().setResponseCode(200); + }); } if ("POST".equals(method) && (permissionUri + "/revoke").equals(resourcePath)) { @@ -3360,19 +3395,45 @@ private MockResponse permissionsApiHandler( if (authorization != null) { return authorization; } - MockResponse validation = - validateResourceAndPrincipal(request.getResource(), request.getPrincipal()); - if (validation != null) { - return validation; - } - permissionStore.remove( - request.getResource(), request.getAccess(), request.getPrincipal()); - return new MockResponse().setResponseCode(200); + return mutatePermission( + request.getResource(), + () -> { + MockResponse validation = + validateResourceAndPrincipal( + request.getResource(), request.getPrincipal()); + if (validation != null) { + return validation; + } + permissionStore.remove( + request.getResource(), request.getAccess(), request.getPrincipal()); + return new MockResponse().setResponseCode(200); + }); } return new MockResponse().setResponseCode(404); } + private MockResponse mutatePermission( + PermissionResource resource, Supplier mutation) { + if (resource.getType() != ResourceType.TABLE && resource.getType() != ResourceType.COLUMN) { + return permissionStore.executeAtomically(mutation); + } + TableMetadata observed = tableMetadata(resource); + if (observed == null) { + return resourceNotFound(resource); + } + synchronized (policyLock(observed.uuid())) { + return permissionStore.executeAtomically( + () -> { + TableMetadata current = tableMetadata(resource); + if (current == null || !observed.uuid().equals(current.uuid())) { + return resourceNotFound(resource); + } + return mutation.get(); + }); + } + } + @Nullable private MockResponse validateResourceAndPrincipal( PermissionResource resource, Map parameters) { @@ -3423,34 +3484,43 @@ private MockResponse validateResource(PermissionResource resource) { case TABLE: case COLUMN: exists = - tableMetadataStore.containsKey( - Identifier.create(resource.getDatabase(), resource.getTable()) - .getFullName()); + databaseStore.containsKey(resource.getDatabase()) + && tableMetadataStore.containsKey( + Identifier.create( + resource.getDatabase(), resource.getTable()) + .getFullName()); break; case FUNCTION: exists = - functionStore.containsKey( - Identifier.create(resource.getDatabase(), resource.getFunction()) - .getFullName()); + databaseStore.containsKey(resource.getDatabase()) + && functionStore.containsKey( + Identifier.create( + resource.getDatabase(), + resource.getFunction()) + .getFullName()); break; case VIEW: exists = - viewStore.containsKey( - Identifier.create(resource.getDatabase(), resource.getView()) - .getFullName()); + databaseStore.containsKey(resource.getDatabase()) + && viewStore.containsKey( + Identifier.create( + resource.getDatabase(), resource.getView()) + .getFullName()); break; default: exists = false; } - return exists - ? null - : mockResponse( - new ErrorResponse( - resource.getType().name(), - resourceName(resource), - "Permission resource does not exist.", - 404), - 404); + return exists ? null : resourceNotFound(resource); + } + + private MockResponse resourceNotFound(PermissionResource resource) { + return mockResponse( + new ErrorResponse( + resource.getType().name(), + resourceName(resource), + "Permission resource does not exist.", + 404), + 404); } @Nullable @@ -3508,6 +3578,50 @@ private static int getPermissionMaxResults(Map parameters) { return Math.max(1, Math.min(maxResults, ListPermissionsRequest.MAX_PAGE_SIZE)); } + private static PagedList buildManagementPage( + List elements, + int maxResults, + @Nullable String pageToken, + java.util.function.Function sortKey) { + String after = decodeManagementPageToken(pageToken); + List remaining = + elements.stream() + .sorted(Comparator.comparing(sortKey)) + .filter( + element -> + after == null + || sortKey.apply(element).compareTo(after) > 0) + .collect(Collectors.toList()); + int end = Math.min(maxResults, remaining.size()); + List page = new ArrayList<>(remaining.subList(0, end)); + String nextPageToken = + end < remaining.size() + ? encodeManagementPageToken(sortKey.apply(page.get(page.size() - 1))) + : null; + return new PagedList<>(page, nextPageToken); + } + + @Nullable + private static String decodeManagementPageToken(@Nullable String pageToken) { + if (pageToken == null) { + return null; + } + String decoded; + try { + decoded = new String(Base64.getUrlDecoder().decode(pageToken), StandardCharsets.UTF_8); + } catch (IllegalArgumentException e) { + throw new IllegalArgumentException("Invalid management page token.", e); + } + checkArgument(decoded.startsWith("v1\0"), "Invalid management page token version."); + return decoded.substring(3); + } + + private static String encodeManagementPageToken(String sortKey) { + return Base64.getUrlEncoder() + .withoutPadding() + .encodeToString(("v1\0" + sortKey).getBytes(StandardCharsets.UTF_8)); + } + private static boolean matches( Map parameters, String key, @Nullable String value) { return !parameters.containsKey(key) || Objects.equals(parameters.get(key), value); @@ -3539,11 +3653,11 @@ private MockResponse policiesApiHandler( if (authorization != null) { return authorization; } - MockResponse resourceError = validateResource(path.resource); - if (resourceError != null) { - return resourceError; + TableMetadata policyTable = tableMetadata(path.resource); + if (policyTable == null) { + return resourceNotFound(path.resource); } - String tableUuid = tableUuid(path.resource); + String tableUuid = policyTable.uuid(); if ("GET".equals(method)) { if (parameters.containsKey("principal")) { MockResponse principalError = validatePrincipal(parameters.get("principal")); @@ -3560,16 +3674,14 @@ private MockResponse policiesApiHandler( Comparator.comparing( policy -> new PolicyKey(tableUuid, policy))) .collect(Collectors.toList()); - int start = - parameters.containsKey(PAGE_TOKEN) - ? Integer.parseInt(parameters.get(PAGE_TOKEN)) - : 0; - int end = Math.min(start + getPermissionMaxResults(parameters), filtered.size()); - String nextPageToken = end < filtered.size() ? String.valueOf(end) : null; + PagedList page = + buildManagementPage( + filtered, + getPermissionMaxResults(parameters), + parameters.get(PAGE_TOKEN), + policy -> new PolicyKey(tableUuid, policy).sortKey()); return mockResponse( - new ListPoliciesResponse( - new ArrayList<>(filtered.subList(start, end)), nextPageToken), - 200); + new ListPoliciesResponse(page.getElements(), page.getNextPageToken()), 200); } if ("POST".equals(method) && !path.drop) { @@ -3693,10 +3805,10 @@ private static String policyResourceName(DropPolicyRequest request) { + (request.getColumn() == null ? "" : ":" + request.getColumn()); } - private String tableUuid(PermissionResource resource) { - return tableMetadataStore - .get(Identifier.create(resource.getDatabase(), resource.getTable()).getFullName()) - .uuid(); + @Nullable + private TableMetadata tableMetadata(PermissionResource resource) { + return tableMetadataStore.get( + Identifier.create(resource.getDatabase(), resource.getTable()).getFullName()); } private Object policyLock(String tableUuid) { @@ -3706,12 +3818,11 @@ private Object policyLock(String tableUuid) { @Nullable private MockResponse validatePolicyTableVersion( PermissionResource resource, String expectedTableUuid) { - MockResponse resourceError = validateResource(resource); - if (resourceError != null) { - return resourceError; + TableMetadata current = tableMetadata(resource); + if (current == null) { + return resourceNotFound(resource); } - String currentTableUuid = tableUuid(resource); - return expectedTableUuid.equals(currentTableUuid) + return expectedTableUuid.equals(current.uuid()) ? null : mockResponse( new ErrorResponse( @@ -3831,6 +3942,20 @@ private void validatePoliciesForSchema( } } + private void validatePermissionsForSchema( + Identifier identifier, TableSchema previous, TableSchema current) { + if (permissionStore.hasColumnAssignments(identifier)) { + checkArgument( + CoreOptions.fromMap(current.options()).queryAuthEnabled(), + "Cannot disable query-auth.enabled while table %s has column permissions.", + identifier.getFullName()); + checkArgument( + permissionStore.canEvolveTableColumns(identifier, previous, current), + "Cannot drop every allowed column while table %s has column permissions.", + identifier.getFullName()); + } + } + private static boolean matchesPolicy(DataPolicy policy, Map parameters) { if (!matches(parameters, "type", policy.type().name())) { return false; diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/RESTPermissionStore.java b/paimon-core/src/test/java/org/apache/paimon/rest/RESTPermissionStore.java index 83d28bc6010b..a1468f2b30f3 100644 --- a/paimon-core/src/test/java/org/apache/paimon/rest/RESTPermissionStore.java +++ b/paimon-core/src/test/java/org/apache/paimon/rest/RESTPermissionStore.java @@ -18,30 +18,43 @@ package org.apache.paimon.rest; +import org.apache.paimon.catalog.Identifier; import org.apache.paimon.management.PermissionAssignment; +import org.apache.paimon.management.PermissionColumns; import org.apache.paimon.management.PermissionResource; +import org.apache.paimon.management.ResourceType; +import org.apache.paimon.schema.TableSchema; +import org.apache.paimon.types.DataField; import java.util.Comparator; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Objects; -import java.util.concurrent.ConcurrentHashMap; +import java.util.function.Function; +import java.util.function.Predicate; +import java.util.function.Supplier; import java.util.stream.Collectors; /** Atomic permission assignment store for the REST catalog test server. */ final class RESTPermissionStore { - private final Map assignments = new ConcurrentHashMap<>(); + private final Map assignments = new HashMap<>(); - void put(PermissionAssignment assignment) { + synchronized T executeAtomically(Supplier operation) { + return operation.get(); + } + + synchronized void put(PermissionAssignment assignment) { assignments.put(PermissionKey.fromAssignment(assignment), assignment); } - void remove(PermissionResource resource, String access, String principal) { + synchronized void remove(PermissionResource resource, String access, String principal) { assignments.remove(new PermissionKey(resource, access, principal)); } - List list(PermissionResource target, Map parameters) { + synchronized List list( + PermissionResource target, Map parameters) { return assignments.values().stream() .filter(assignment -> assignment.getResource().equals(target)) .filter(assignment -> matches(parameters, "principal", assignment.getPrincipal())) @@ -50,29 +63,192 @@ List list(PermissionResource target, Map p .collect(Collectors.toList()); } + synchronized void renameTable(Identifier source, Identifier destination) { + replaceResources( + resource -> + isTableResource(resource, source, ResourceType.TABLE) + || isTableResource(resource, source, ResourceType.COLUMN), + resource -> + new PermissionResource( + resource.getType(), + destination.getDatabaseName(), + destination.getTableName(), + null, + null)); + } + + synchronized void renameView(Identifier source, Identifier destination) { + replaceResources( + resource -> + resource.getType() == ResourceType.VIEW + && Objects.equals(source.getDatabaseName(), resource.getDatabase()) + && Objects.equals(source.getObjectName(), resource.getView()), + resource -> + new PermissionResource( + ResourceType.VIEW, + destination.getDatabaseName(), + null, + null, + destination.getObjectName())); + } + + synchronized void removeTable(Identifier identifier) { + removeResources( + resource -> + isTableResource(resource, identifier, ResourceType.TABLE) + || isTableResource(resource, identifier, ResourceType.COLUMN)); + } + + synchronized void removeView(Identifier identifier) { + removeResources( + resource -> + resource.getType() == ResourceType.VIEW + && Objects.equals( + identifier.getDatabaseName(), resource.getDatabase()) + && Objects.equals(identifier.getObjectName(), resource.getView())); + } + + synchronized void removeFunction(Identifier identifier) { + removeResources( + resource -> + resource.getType() == ResourceType.FUNCTION + && Objects.equals( + identifier.getDatabaseName(), resource.getDatabase()) + && Objects.equals( + identifier.getObjectName(), resource.getFunction())); + } + + synchronized void removeDatabase(String database) { + removeResources(resource -> Objects.equals(database, resource.getDatabase())); + } + + synchronized boolean hasColumnAssignments(Identifier identifier) { + return assignments.values().stream() + .map(PermissionAssignment::getResource) + .anyMatch(resource -> isTableResource(resource, identifier, ResourceType.COLUMN)); + } + + synchronized boolean canEvolveTableColumns( + Identifier identifier, TableSchema previous, TableSchema current) { + Map currentNamesByPreviousName = + currentNamesByPreviousName(previous, current); + return assignments.values().stream() + .filter( + assignment -> + isTableResource( + assignment.getResource(), identifier, ResourceType.COLUMN)) + .map(PermissionAssignment::getColumns) + .allMatch( + columns -> + columns.getColumnNames() == null + || columns.getColumnNames().stream() + .anyMatch( + name -> + currentNamesByPreviousName.get(name) + != null)); + } + + synchronized void evolveTableColumns( + Identifier identifier, TableSchema previous, TableSchema current) { + Map currentNamesByPreviousName = + currentNamesByPreviousName(previous, current); + List columnAssignments = + assignments.values().stream() + .filter( + assignment -> + isTableResource( + assignment.getResource(), + identifier, + ResourceType.COLUMN)) + .collect(Collectors.toList()); + for (PermissionAssignment assignment : columnAssignments) { + PermissionColumns columns = assignment.getColumns(); + List source = + columns.getColumnNames() == null + ? columns.getExcludedColumnNames() + : columns.getColumnNames(); + List evolved = + source.stream() + .map(currentNamesByPreviousName::get) + .filter(Objects::nonNull) + .collect(Collectors.toList()); + assignments.remove(PermissionKey.fromAssignment(assignment)); + if (!evolved.isEmpty()) { + PermissionColumns evolvedColumns = + columns.getColumnNames() == null + ? new PermissionColumns(null, evolved) + : new PermissionColumns(evolved, null); + PermissionAssignment evolvedAssignment = + new PermissionAssignment( + assignment.getResource(), + assignment.getAccess(), + assignment.getPrincipal(), + evolvedColumns, + assignment.getExpireTime()); + assignments.put(PermissionKey.fromAssignment(evolvedAssignment), evolvedAssignment); + } + } + } + + private static Map currentNamesByPreviousName( + TableSchema previous, TableSchema current) { + Map currentNamesById = + current.fields().stream().collect(Collectors.toMap(DataField::id, DataField::name)); + Map result = new HashMap<>(); + for (DataField field : previous.fields()) { + result.put(field.name(), currentNamesById.get(field.id())); + } + return result; + } + private static boolean matches(Map parameters, String key, String value) { return !parameters.containsKey(key) || parameters.get(key).equals(value); } - private static String sortKey(PermissionAssignment assignment) { + static String sortKey(PermissionAssignment assignment) { PermissionResource source = assignment.getResource(); - return source.getType().name() - + '\0' - + value(source.getDatabase()) - + '\0' - + value(source.getTable()) - + '\0' - + value(source.getFunction()) - + '\0' - + value(source.getView()) - + '\0' - + assignment.getAccess() - + '\0' - + assignment.getPrincipal(); - } - - private static String value(String value) { - return value == null ? "" : value; + return cursorPart(source.getType().name()) + + cursorPart(source.getDatabase()) + + cursorPart(source.getTable()) + + cursorPart(source.getFunction()) + + cursorPart(source.getView()) + + cursorPart(assignment.getAccess()) + + cursorPart(assignment.getPrincipal()); + } + + private static String cursorPart(String value) { + return value == null ? "-1:" : value.length() + ":" + value; + } + + private void replaceResources( + Predicate matches, + Function replacement) { + List replaced = + assignments.values().stream() + .filter(assignment -> matches.test(assignment.getResource())) + .collect(Collectors.toList()); + for (PermissionAssignment assignment : replaced) { + assignments.remove(PermissionKey.fromAssignment(assignment)); + PermissionAssignment newAssignment = + new PermissionAssignment( + replacement.apply(assignment.getResource()), + assignment.getAccess(), + assignment.getPrincipal(), + assignment.getColumns(), + assignment.getExpireTime()); + assignments.put(PermissionKey.fromAssignment(newAssignment), newAssignment); + } + } + + private void removeResources(Predicate matches) { + assignments.entrySet().removeIf(entry -> matches.test(entry.getValue().getResource())); + } + + private static boolean isTableResource( + PermissionResource resource, Identifier identifier, ResourceType type) { + return resource.getType() == type + && Objects.equals(identifier.getDatabaseName(), resource.getDatabase()) + && Objects.equals(identifier.getTableName(), resource.getTable()); } private static class PermissionKey { diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/RESTPermissionStoreTest.java b/paimon-core/src/test/java/org/apache/paimon/rest/RESTPermissionStoreTest.java index 73dfacb12976..3d51004872b9 100644 --- a/paimon-core/src/test/java/org/apache/paimon/rest/RESTPermissionStoreTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/rest/RESTPermissionStoreTest.java @@ -18,17 +18,25 @@ package org.apache.paimon.rest; +import org.apache.paimon.catalog.Identifier; import org.apache.paimon.management.PermissionAssignment; import org.apache.paimon.management.PermissionColumns; import org.apache.paimon.management.PermissionResource; import org.apache.paimon.management.ResourceType; +import org.apache.paimon.schema.TableSchema; +import org.apache.paimon.types.DataField; +import org.apache.paimon.types.DataTypes; import org.junit.jupiter.api.Test; import java.time.Instant; import java.util.Arrays; +import java.util.Collections; import java.util.HashMap; import java.util.Map; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; import java.util.stream.IntStream; import static org.assertj.core.api.Assertions.assertThat; @@ -57,6 +65,42 @@ void testConcurrentGrantReplacesTheSameIdentity() { assertThat(store.list(table, tableParameters())).hasSize(1); } + @Test + void testAtomicMutationCannotRaceLifecycleCleanup() throws Exception { + RESTPermissionStore store = new RESTPermissionStore(); + Identifier identifier = Identifier.create("sales", "orders"); + PermissionAssignment assignment = + new PermissionAssignment(tableResource(), "SELECT", ANALYST, null); + CountDownLatch mutationStarted = new CountDownLatch(1); + CountDownLatch releaseMutation = new CountDownLatch(1); + CountDownLatch cleanupStarted = new CountDownLatch(1); + + CompletableFuture mutation = + CompletableFuture.runAsync( + () -> + store.executeAtomically( + () -> { + mutationStarted.countDown(); + await(releaseMutation); + store.put(assignment); + return null; + })); + assertThat(mutationStarted.await(10, TimeUnit.SECONDS)).isTrue(); + CompletableFuture cleanup = + CompletableFuture.runAsync( + () -> { + cleanupStarted.countDown(); + store.removeTable(identifier); + }); + assertThat(cleanupStarted.await(10, TimeUnit.SECONDS)).isTrue(); + assertThat(cleanup).isNotDone(); + + releaseMutation.countDown(); + CompletableFuture.allOf(mutation, cleanup).get(10, TimeUnit.SECONDS); + + assertThat(store.list(tableResource(), Collections.emptyMap())).isEmpty(); + } + @Test void testListReturnsOnlyTheExactTarget() { RESTPermissionStore store = new RESTPermissionStore(); @@ -71,6 +115,25 @@ void testListReturnsOnlyTheExactTarget() { .isEqualTo("SELECT"); } + @Test + void testCursorKeyDoesNotFlattenOpaqueResourceParts() { + PermissionAssignment first = + new PermissionAssignment( + new PermissionResource(ResourceType.TABLE, "a", "b\0c", null, null), + "SELECT", + ANALYST, + null); + PermissionAssignment second = + new PermissionAssignment( + new PermissionResource(ResourceType.TABLE, "a\0b", "c", null, null), + "SELECT", + ANALYST, + null); + + assertThat(RESTPermissionStore.sortKey(first)) + .isNotEqualTo(RESTPermissionStore.sortKey(second)); + } + @Test void testColumnGrantReplacesTheWholeColumnRangeForTheSameIdentity() { RESTPermissionStore store = new RESTPermissionStore(); @@ -97,6 +160,112 @@ void testColumnGrantReplacesTheWholeColumnRangeForTheSameIdentity() { .isEqualTo(Arrays.asList("email")); } + @Test + void testTableAndColumnAssignmentsFollowResourceAndSchemaLifecycle() { + RESTPermissionStore store = new RESTPermissionStore(); + PermissionResource table = tableResource(); + PermissionResource column = columnResource(); + store.put(new PermissionAssignment(table, "SELECT", ANALYST, null)); + store.put( + new PermissionAssignment( + column, + "SELECT", + ANALYST, + new PermissionColumns(null, Collections.singletonList("email")), + null)); + + Identifier source = Identifier.create("sales", "orders"); + Identifier destination = Identifier.create("sales", "renamed_orders"); + store.renameTable(source, destination); + + PermissionResource renamedTable = + new PermissionResource(ResourceType.TABLE, "sales", "renamed_orders", null, null); + PermissionResource renamedColumn = + new PermissionResource(ResourceType.COLUMN, "sales", "renamed_orders", null, null); + assertThat(store.list(table, Collections.emptyMap())).isEmpty(); + assertThat(store.list(renamedTable, Collections.emptyMap())) + .singleElement() + .extracting(PermissionAssignment::getResource) + .isEqualTo(renamedTable); + + TableSchema original = tableSchema(new DataField(1, "email", DataTypes.STRING())); + TableSchema renamed = tableSchema(new DataField(1, "contact", DataTypes.STRING())); + store.evolveTableColumns(destination, original, renamed); + assertThat(store.list(renamedColumn, Collections.emptyMap())) + .singleElement() + .extracting(PermissionAssignment::getColumns) + .extracting(PermissionColumns::getExcludedColumnNames) + .isEqualTo(Collections.singletonList("contact")); + + store.put( + new PermissionAssignment( + renamedColumn, + "SELECT", + ANALYST, + new PermissionColumns(Collections.singletonList("contact"), null), + null)); + assertThat(store.canEvolveTableColumns(destination, renamed, tableSchema())).isFalse(); + store.put( + new PermissionAssignment( + renamedColumn, + "SELECT", + ANALYST, + new PermissionColumns(null, Collections.singletonList("contact")), + null)); + store.evolveTableColumns(destination, renamed, tableSchema()); + assertThat(store.list(renamedColumn, Collections.emptyMap())).isEmpty(); + + store.removeTable(destination); + assertThat(store.list(renamedTable, Collections.emptyMap())).isEmpty(); + } + + @Test + void testDroppingDatabaseRemovesDirectAndChildAssignments() { + RESTPermissionStore store = new RESTPermissionStore(); + PermissionResource catalog = + new PermissionResource(ResourceType.CATALOG, null, null, null, null); + PermissionResource database = + new PermissionResource(ResourceType.DATABASE, "sales", null, null, null); + PermissionResource databaseAll = + new PermissionResource(ResourceType.DATABASE_ALL, "sales", null, null, null); + PermissionResource view = + new PermissionResource(ResourceType.VIEW, "sales", null, null, "orders_view"); + PermissionResource function = + new PermissionResource(ResourceType.FUNCTION, "sales", null, "orders_fn", null); + for (PermissionResource resource : + Arrays.asList(catalog, database, databaseAll, tableResource(), view, function)) { + store.put(new PermissionAssignment(resource, "ALL", ANALYST, null)); + } + + store.removeDatabase("sales"); + + for (PermissionResource resource : + Arrays.asList(database, databaseAll, tableResource(), view, function)) { + assertThat(store.list(resource, Collections.emptyMap())).isEmpty(); + } + assertThat(store.list(catalog, Collections.emptyMap())).hasSize(1); + } + + @Test + void testViewAssignmentFollowsRenameAndDrop() { + RESTPermissionStore store = new RESTPermissionStore(); + Identifier source = Identifier.create("sales", "orders_view"); + Identifier destination = Identifier.create("sales", "renamed_view"); + PermissionResource sourceResource = + new PermissionResource(ResourceType.VIEW, "sales", null, null, "orders_view"); + PermissionResource destinationResource = + new PermissionResource(ResourceType.VIEW, "sales", null, null, "renamed_view"); + store.put(new PermissionAssignment(sourceResource, "SELECT", ANALYST, null)); + + store.renameView(source, destination); + + assertThat(store.list(sourceResource, Collections.emptyMap())).isEmpty(); + assertThat(store.list(destinationResource, Collections.emptyMap())).hasSize(1); + + store.removeView(destination); + assertThat(store.list(destinationResource, Collections.emptyMap())).isEmpty(); + } + private static PermissionResource tableResource() { return new PermissionResource(ResourceType.TABLE, "sales", "orders", null, null); } @@ -120,4 +289,25 @@ private static Map columnParameters() { parameters.put("table", "orders"); return parameters; } + + private static TableSchema tableSchema(DataField... fields) { + int highestFieldId = Arrays.stream(fields).mapToInt(DataField::id).max().orElse(0); + return new TableSchema( + 1, + Arrays.asList(fields), + highestFieldId, + Collections.emptyList(), + Collections.emptyList(), + Collections.emptyMap(), + null); + } + + private static void await(CountDownLatch latch) { + try { + latch.await(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new RuntimeException(e); + } + } } diff --git a/paimon-core/src/test/java/org/apache/paimon/table/source/AbstractDataTableReadTest.java b/paimon-core/src/test/java/org/apache/paimon/table/source/AbstractDataTableReadTest.java new file mode 100644 index 000000000000..b76c42be202e --- /dev/null +++ b/paimon-core/src/test/java/org/apache/paimon/table/source/AbstractDataTableReadTest.java @@ -0,0 +1,136 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.table.source; + +import org.apache.paimon.catalog.TableQueryAuthResult; +import org.apache.paimon.data.InternalRow; +import org.apache.paimon.predicate.FieldRef; +import org.apache.paimon.predicate.Predicate; +import org.apache.paimon.predicate.UpperTransform; +import org.apache.paimon.reader.RecordReader; +import org.apache.paimon.schema.TableSchema; +import org.apache.paimon.types.DataField; +import org.apache.paimon.types.DataTypes; +import org.apache.paimon.types.RowType; +import org.apache.paimon.utils.JsonSerdeUtil; + +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.util.Arrays; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.Map; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.mock; + +/** Tests query-authorization projection expansion in {@link AbstractDataTableRead}. */ +class AbstractDataTableReadTest { + + @Test + void testMaskDependenciesPreserveNestedProjectionAndSkipUnselectedMasks() throws IOException { + RowType fullProfile = + new RowType( + Arrays.asList( + new DataField(1, "a", DataTypes.INT()), + new DataField(2, "b", DataTypes.STRING()))); + DataField profile = new DataField(0, "profile", fullProfile); + DataField protectedField = new DataField(3, "protected", DataTypes.STRING()); + DataField seed = new DataField(4, "seed", DataTypes.STRING()); + DataField unused = new DataField(5, "unused", DataTypes.STRING()); + DataField unusedSeed = new DataField(6, "unused_seed", DataTypes.STRING()); + TableSchema schema = + new TableSchema( + 1, + Arrays.asList(profile, protectedField, seed, unused, unusedSeed), + 6, + Collections.emptyList(), + Collections.emptyList(), + Collections.emptyMap(), + null); + + RowType prunedProfile = + new RowType(Collections.singletonList(new DataField(2, "b", DataTypes.STRING()))); + RowType requestedType = + new RowType(Arrays.asList(profile.newType(prunedProfile), protectedField)); + TestingDataTableRead read = new TestingDataTableRead(schema); + read.withReadType(requestedType); + + Map masks = new LinkedHashMap<>(); + masks.put( + "protected", + JsonSerdeUtil.toFlatJson( + new UpperTransform( + Collections.singletonList( + new FieldRef(4, "seed", DataTypes.STRING()))))); + masks.put( + "unused", + JsonSerdeUtil.toFlatJson( + new UpperTransform( + Collections.singletonList( + new FieldRef(6, "unused_seed", DataTypes.STRING()))))); + + read.createAuthedReader(new TableQueryAuthResult(null, masks)); + + assertThat(read.appliedReadType().getFieldNames()) + .containsExactly("profile", "protected", "seed"); + assertThat(read.appliedReadType().getTypeAt(0)).isEqualTo(prunedProfile); + } + + private static class TestingDataTableRead extends AbstractDataTableRead { + + private RowType appliedReadType; + + private TestingDataTableRead(TableSchema schema) { + super(schema); + } + + @Override + public void applyReadType(RowType readType) { + appliedReadType = readType; + } + + @Override + public RecordReader reader(Split split) { + return new RecordReader() { + @Override + public RecordIterator readBatch() { + return null; + } + + @Override + public void close() {} + }; + } + + @Override + protected InnerTableRead innerWithFilter(Predicate predicate) { + return this; + } + + private void createAuthedReader(TableQueryAuthResult authResult) throws IOException { + createDataReader(mock(Split.class), authResult); + } + + private RowType appliedReadType() { + return appliedReadType; + } + } +} From 214c327622ca40f41a1f86108cbc279194c70dae Mon Sep 17 00:00:00 2001 From: JingsongLi Date: Wed, 26 Aug 2026 19:59:00 +0800 Subject: [PATCH 4/5] [fix] Harden query authorization lifecycle handling --- .../table/source/AbstractDataTableRead.java | 4 + .../paimon/rest/MockRESTCatalogTest.java | 214 ++++++ .../apache/paimon/rest/RESTCatalogServer.java | 716 +++++++----------- .../paimon/rest/RESTCatalogServerUtils.java | 380 ++++++++++ .../rest/RESTColumnPermissionSupport.java | 4 +- .../paimon/rest/RESTPermissionStoreTest.java | 29 + .../source/AbstractDataTableReadTest.java | 4 + 7 files changed, 902 insertions(+), 449 deletions(-) create mode 100644 paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogServerUtils.java diff --git a/paimon-core/src/main/java/org/apache/paimon/table/source/AbstractDataTableRead.java b/paimon-core/src/main/java/org/apache/paimon/table/source/AbstractDataTableRead.java index 9fbb5827813e..9bf4982552dc 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/source/AbstractDataTableRead.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/source/AbstractDataTableRead.java @@ -123,6 +123,10 @@ protected final QueryAuthContext unwrapQueryAuthSplit(Split split) { protected final RecordReader createDataReader( Split split, @Nullable TableQueryAuthResult authResult) throws IOException { + // A TableRead can be reused for multiple splits. Authentication may have expanded the + // physical projection for the previous split, so always restore the logical projection + // before applying the current split's authorization dependencies. + applyReadType(currentReadType()); RecordReader reader; if (authResult == null) { reader = reader(split); diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/MockRESTCatalogTest.java b/paimon-core/src/test/java/org/apache/paimon/rest/MockRESTCatalogTest.java index 093c8f7bf5cc..bd7fedce3ae3 100644 --- a/paimon-core/src/test/java/org/apache/paimon/rest/MockRESTCatalogTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/rest/MockRESTCatalogTest.java @@ -28,6 +28,11 @@ import org.apache.paimon.catalog.TableQueryAuthResult; import org.apache.paimon.fs.Path; import org.apache.paimon.fs.local.LocalFileIO; +import org.apache.paimon.management.ListPermissionsRequest; +import org.apache.paimon.management.PermissionAssignment; +import org.apache.paimon.management.PermissionColumns; +import org.apache.paimon.management.PermissionResource; +import org.apache.paimon.management.ResourceType; import org.apache.paimon.options.CatalogOptions; import org.apache.paimon.options.Options; import org.apache.paimon.partition.Partition; @@ -46,20 +51,24 @@ import org.apache.paimon.rest.auth.DLFTokenLoader; import org.apache.paimon.rest.auth.DLFTokenLoaderFactory; import org.apache.paimon.rest.auth.RESTAuthParameter; +import org.apache.paimon.rest.exceptions.BadRequestException; import org.apache.paimon.rest.exceptions.NotAuthorizedException; import org.apache.paimon.rest.exceptions.NotImplementedException; import org.apache.paimon.rest.responses.ConfigResponse; import org.apache.paimon.schema.Schema; +import org.apache.paimon.schema.SchemaChange; import org.apache.paimon.table.BlobDescriptorReaderFactory; import org.apache.paimon.table.FileStoreTable; import org.apache.paimon.table.FormatTable; import org.apache.paimon.table.format.FormatTablePartitionManager; +import org.apache.paimon.types.DataField; import org.apache.paimon.types.DataTypes; import org.apache.paimon.types.RowType; import org.apache.paimon.utils.InstantiationUtil; import org.apache.paimon.utils.JsonSerdeUtil; import org.apache.paimon.shade.guava30.com.google.common.collect.ImmutableMap; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonGetter; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; @@ -83,6 +92,7 @@ import static org.assertj.core.api.Assertions.assertThatExceptionOfType; import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; /** Test REST Catalog on Mocked REST server. */ class MockRESTCatalogTest extends RESTCatalogTest { @@ -132,6 +142,180 @@ void testAuthFail() { .isInstanceOf(NotAuthorizedException.class); } + @Test + void testInvalidManagementDtoReturnsBadRequest() throws Exception { + String database = "invalid_management_dto"; + Identifier identifier = Identifier.create(database, "orders"); + catalog.createDatabase(database, false); + catalog.createTable( + identifier, + Schema.newBuilder() + .column("id", DataTypes.INT()) + .option(CoreOptions.QUERY_AUTH_ENABLED.key(), "true") + .build(), + false); + + BadRequestException error = + assertThrows( + BadRequestException.class, + () -> + new HttpClient(restCatalogServer.getUrl()) + .post( + new ResourcePaths("paimon").grantPermission(), + new InvalidColumnGrantRequest(identifier), + restCatalog.api().authFunction())); + + assertThat(error).hasMessageContaining("columns is required for COLUMN resource"); + } + + @Test + void testCreateResourceRejectsDatabaseMismatch() throws Exception { + String pathDatabase = "path_database"; + String bodyDatabase = MockRESTMessage.databaseName(); + catalog.createDatabase(pathDatabase, true); + catalog.createDatabase(bodyDatabase, true); + HttpClient client = new HttpClient(restCatalogServer.getUrl()); + ResourcePaths paths = new ResourcePaths("paimon"); + + BadRequestException tableError = + assertThrows( + BadRequestException.class, + () -> + client.post( + paths.tables(pathDatabase), + MockRESTMessage.createTableRequest("orders"), + restCatalog.api().authFunction())); + BadRequestException viewError = + assertThrows( + BadRequestException.class, + () -> + client.post( + paths.views(pathDatabase), + MockRESTMessage.createViewRequest("orders_view"), + restCatalog.api().authFunction())); + + assertThat(tableError) + .hasMessageContaining( + "The database in the table identifier must match the request path"); + assertThat(viewError) + .hasMessageContaining( + "The database in the view identifier must match the request path"); + assertThat(catalog.listTables(bodyDatabase)).isEmpty(); + assertThat(catalog.listViews(bodyDatabase)).isEmpty(); + } + + @Test + void testRenamePreservesHighestFieldIdForColumnPermissions() throws Exception { + String principal = "analyst"; + Identifier source = Identifier.create("schema_lifecycle_db", "orders"); + Identifier destination = Identifier.create("schema_lifecycle_db", "renamed_orders"); + catalog.createDatabase(source.getDatabaseName(), false); + catalog.createTable( + source, + new Schema( + Arrays.asList( + new DataField(0, "id", DataTypes.INT()), + new DataField(1, "removed", DataTypes.STRING()), + new DataField(2, "secret", DataTypes.STRING())), + Collections.emptyList(), + Collections.emptyList(), + Collections.singletonMap(CoreOptions.QUERY_AUTH_ENABLED.key(), "true"), + null), + false); + restCatalogServer.registerManagementPrincipal(principal); + restCatalog + .permissionManagement() + .grantPermission( + new PermissionAssignment( + new PermissionResource( + ResourceType.COLUMN, + source.getDatabaseName(), + source.getTableName(), + null, + null), + "SELECT", + principal, + new PermissionColumns(Collections.singletonList("id"), null), + null)); + + catalog.alterTable( + source, Collections.singletonList(SchemaChange.dropColumn("removed")), false); + catalog.renameTable(source, destination, false); + catalog.alterTable( + destination, + Collections.singletonList(SchemaChange.addColumn("new_column", DataTypes.STRING())), + false); + + assertThat(catalog.getTable(destination).rowType().getFields()) + .extracting(DataField::id) + .containsExactly(0, 2, 3); + } + + @Test + void testDropDatabaseRemovesViewsFunctionsAndAssignments() throws Exception { + String database = "cascade_management_db"; + String principal = "analyst"; + Identifier view = Identifier.create(database, "orders_view"); + Identifier function = Identifier.create(database, "orders_function"); + catalog.createDatabase(database, false); + catalog.createView(view, createView(view), false); + catalog.createFunction(function, MockRESTMessage.function(function), false); + restCatalogServer.registerManagementPrincipal(principal); + PermissionResource viewResource = + new PermissionResource( + ResourceType.VIEW, database, null, null, view.getObjectName()); + PermissionResource functionResource = + new PermissionResource( + ResourceType.FUNCTION, database, null, function.getObjectName(), null); + restCatalog + .permissionManagement() + .grantPermission(new PermissionAssignment(viewResource, "SELECT", principal, null)); + restCatalog + .permissionManagement() + .grantPermission( + new PermissionAssignment(functionResource, "SELECT", principal, null)); + + catalog.dropDatabase(database, false, true); + catalog.createDatabase(database, false); + + assertThat(catalog.listViews(database)).isEmpty(); + assertThat(catalog.listFunctions(database)).isEmpty(); + catalog.createView(view, createView(view), false); + catalog.createFunction(function, MockRESTMessage.function(function), false); + assertThat( + restCatalog + .permissionManagement() + .listPermissions( + new ListPermissionsRequest( + ResourceType.VIEW, + database, + null, + null, + view.getObjectName(), + null, + null, + null, + null)) + .getElements()) + .isEmpty(); + assertThat( + restCatalog + .permissionManagement() + .listPermissions( + new ListPermissionsRequest( + ResourceType.FUNCTION, + database, + null, + function.getObjectName(), + null, + null, + null, + null, + null)) + .getElements()) + .isEmpty(); + } + @Test void testDlfStSTokenAuth() throws Exception { String akId = "akId" + UUID.randomUUID(); @@ -854,6 +1038,36 @@ private RESTCatalog initCatalogUtil( return new RESTCatalog(CatalogContext.create(options)); } + private static class InvalidColumnGrantRequest implements RESTRequest { + + private final PermissionResource resource; + + private InvalidColumnGrantRequest(Identifier identifier) { + this.resource = + new PermissionResource( + ResourceType.COLUMN, + identifier.getDatabaseName(), + identifier.getTableName(), + null, + null); + } + + @JsonGetter("resource") + public PermissionResource getResource() { + return resource; + } + + @JsonGetter("access") + public String getAccess() { + return "SELECT"; + } + + @JsonGetter("principal") + public String getPrincipal() { + return "analyst"; + } + } + private static String extractHost(String uri) { String withoutProtocol = uri.replaceFirst("^https?://", ""); int pathIndex = withoutProtocol.indexOf('/'); diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogServer.java b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogServer.java index 28d93c4f6ffb..8e36aa384401 100644 --- a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogServer.java +++ b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogServer.java @@ -42,7 +42,6 @@ import org.apache.paimon.function.FunctionImpl; import org.apache.paimon.management.ColumnMask; import org.apache.paimon.management.DataPolicy; -import org.apache.paimon.management.ListPermissionsRequest; import org.apache.paimon.management.PermissionAssignment; import org.apache.paimon.management.PermissionResource; import org.apache.paimon.management.ResourceType; @@ -52,9 +51,10 @@ import org.apache.paimon.partition.Partition; import org.apache.paimon.partition.PartitionStatistics; import org.apache.paimon.partition.PartitionUtils; -import org.apache.paimon.predicate.FieldRef; import org.apache.paimon.predicate.Predicate; import org.apache.paimon.predicate.Transform; +import org.apache.paimon.rest.RESTCatalogServerUtils.InvalidRequestException; +import org.apache.paimon.rest.RESTCatalogServerUtils.PolicyPath; import org.apache.paimon.rest.auth.AuthProvider; import org.apache.paimon.rest.auth.RESTAuthParameter; import org.apache.paimon.rest.requests.AlterDatabaseRequest; @@ -158,12 +158,10 @@ import java.io.FileNotFoundException; import java.io.IOException; import java.io.UncheckedIOException; -import java.nio.charset.StandardCharsets; import java.time.Duration; import java.time.ZoneId; import java.util.ArrayList; import java.util.Arrays; -import java.util.Base64; import java.util.Collections; import java.util.Comparator; import java.util.HashMap; @@ -198,6 +196,22 @@ import static org.apache.paimon.rest.RESTApi.TABLE_TYPE; import static org.apache.paimon.rest.RESTApi.TAG_NAME_PREFIX; import static org.apache.paimon.rest.RESTApi.VIEW_NAME_PATTERN; +import static org.apache.paimon.rest.RESTCatalogServerUtils.buildManagementPage; +import static org.apache.paimon.rest.RESTCatalogServerUtils.canonicalizePolicy; +import static org.apache.paimon.rest.RESTCatalogServerUtils.errorMessage; +import static org.apache.paimon.rest.RESTCatalogServerUtils.findCause; +import static org.apache.paimon.rest.RESTCatalogServerUtils.getPermissionMaxResults; +import static org.apache.paimon.rest.RESTCatalogServerUtils.matchesPolicy; +import static org.apache.paimon.rest.RESTCatalogServerUtils.parseColumnMask; +import static org.apache.paimon.rest.RESTCatalogServerUtils.parseRequest; +import static org.apache.paimon.rest.RESTCatalogServerUtils.parseRowFilter; +import static org.apache.paimon.rest.RESTCatalogServerUtils.permissionResource; +import static org.apache.paimon.rest.RESTCatalogServerUtils.policyPath; +import static org.apache.paimon.rest.RESTCatalogServerUtils.policyResourceName; +import static org.apache.paimon.rest.RESTCatalogServerUtils.resourceName; +import static org.apache.paimon.rest.RESTCatalogServerUtils.validatePermissionsForSchema; +import static org.apache.paimon.rest.RESTCatalogServerUtils.validatePoliciesForSchema; +import static org.apache.paimon.rest.RESTCatalogServerUtils.withResource; import static org.apache.paimon.rest.ResourcePaths.FUNCTIONS; import static org.apache.paimon.rest.ResourcePaths.FUNCTION_DETAILS; import static org.apache.paimon.rest.ResourcePaths.TABLE_DETAILS; @@ -225,6 +239,7 @@ public class RESTCatalogServer { private final RESTPermissionStore permissionStore = new RESTPermissionStore(); private final Map policyStore = new ConcurrentHashMap<>(); private final Map tablePolicyLocks = new ConcurrentHashMap<>(); + private final Map databaseLifecycleLocks = new ConcurrentHashMap<>(); private final TableLifecycleLocks tableLifecycleLocks = new TableLifecycleLocks(); private final Set managementPrincipals = new HashSet<>(); private final Set queryPrincipals = new HashSet<>(); @@ -610,7 +625,7 @@ && isTableByIdRequest(request.getPath())) { } if (isMarkDonePartitions) { MarkDonePartitionsRequest markDonePartitionsRequest = - RESTApi.fromJson(data, MarkDonePartitionsRequest.class); + parseRequest(data, MarkDonePartitionsRequest.class); catalog.markDonePartitions( identifier, markDonePartitionsRequest.getPartitionSpecs()); return new MockResponse().setResponseCode(200); @@ -629,12 +644,12 @@ && isTableByIdRequest(request.getPath())) { identifier); } else if (isListPartitionsByFilter) { ListPartitionsByFilterRequest listPartitionsByFilterRequest = - RESTApi.fromJson(data, ListPartitionsByFilterRequest.class); + parseRequest(data, ListPartitionsByFilterRequest.class); return listPartitionsByFilter( identifier, listPartitionsByFilterRequest); } else if (isListPartitionsByNames) { ListPartitionsByNamesRequest listPartitionsByNamesRequest = - RESTApi.fromJson(data, ListPartitionsByNamesRequest.class); + parseRequest(data, ListPartitionsByNamesRequest.class); return listPartitionsByNames( parameters, identifier, @@ -670,7 +685,7 @@ && isTableByIdRequest(request.getPath())) { return commitTableHandle(identifier, restAuthParameter.data()); } else if (isRollbackTable) { RollbackTableRequest requestBody = - RESTApi.fromJson(data, RollbackTableRequest.class); + parseRequest(data, RollbackTableRequest.class); if (noPermissionTables.contains(identifier.getFullName())) { throw new Catalog.TableNoPermissionException(identifier); } @@ -873,28 +888,28 @@ && isTableByIdRequest(request.getPath())) { e.getMessage(), 409); return mockResponse(response, 409); + } catch (InvalidRequestException e) { + response = new ErrorResponse(null, null, e.getMessage(), 400); + return mockResponse(response, 400); } catch (IllegalArgumentException e) { response = new ErrorResponse(null, null, e.getMessage(), 400); return mockResponse(response, 400); } catch (Exception e) { e.printStackTrace(); - if (e.getCause() instanceof IllegalArgumentException) { + Throwable invalidArgument = findCause(e, IllegalArgumentException.class); + Throwable jsonProcessing = findCause(e, JsonProcessingException.class); + if (invalidArgument != null && jsonProcessing == null) { response = - new ErrorResponse( - null, null, e.getCause().getCause().getMessage(), 400); + new ErrorResponse(null, null, errorMessage(invalidArgument), 400); return mockResponse(response, 400); - } else if (e instanceof UnsupportedOperationException - || e.getCause() instanceof UnsupportedOperationException) { - response = new ErrorResponse(null, null, e.getMessage(), 501); + } else if (findCause(e, UnsupportedOperationException.class) != null) { + response = new ErrorResponse(null, null, errorMessage(e), 501); return mockResponse(response, 501); - } else if (e instanceof IllegalStateException - || e.getCause() instanceof IllegalStateException) { - response = new ErrorResponse(null, null, e.getMessage(), 500); + } else if (findCause(e, IllegalStateException.class) != null) { + response = new ErrorResponse(null, null, errorMessage(e), 500); return mockResponse(response, 500); } - return new MockResponse() - .setResponseCode(500) - .setBody(e.getCause().getMessage()); + return new MockResponse().setResponseCode(500).setBody(errorMessage(e)); } } }; @@ -979,7 +994,7 @@ private MockResponse listConsumers(Identifier identifier) throws Exception { } private MockResponse resetConsumer(Identifier identifier, String data) throws Exception { - ResetConsumerRequest request = RESTApi.fromJson(data, ResetConsumerRequest.class); + ResetConsumerRequest request = parseRequest(data, ResetConsumerRequest.class); FileStoreTable table = (FileStoreTable) catalog.getTable(identifier); ConsumerManager consumerManager = new ConsumerManager(table.fileIO(), table.location(), "main"); @@ -1048,7 +1063,7 @@ private Optional checkTablePartitioned(Identifier identifier) { } private MockResponse authTable(Identifier identifier, String data) throws Exception { - AuthTableQueryRequest requestBody = RESTApi.fromJson(data, AuthTableQueryRequest.class); + AuthTableQueryRequest requestBody = parseRequest(data, AuthTableQueryRequest.class); if (noPermissionTables.contains(identifier.getFullName())) { throw new Catalog.TableNoPermissionException(identifier); } @@ -1157,7 +1172,7 @@ private boolean appliesToQueryPrincipal(DataPolicy policy) { } private MockResponse commitTableHandle(Identifier identifier, String data) throws Exception { - CommitTableRequest requestBody = RESTApi.fromJson(data, CommitTableRequest.class); + CommitTableRequest requestBody = parseRequest(data, CommitTableRequest.class); if (noPermissionTables.contains(identifier.getFullName())) { throw new Catalog.TableNoPermissionException(identifier); } @@ -1230,7 +1245,7 @@ private MockResponse rollbackTableByTagNameHandle(Identifier identifier, String } private MockResponse rollbackSchemaHandle(Identifier identifier, String data) throws Exception { - RollbackSchemaRequest requestBody = RESTApi.fromJson(data, RollbackSchemaRequest.class); + RollbackSchemaRequest requestBody = parseRequest(data, RollbackSchemaRequest.class); if (noPermissionTables.contains(identifier.getFullName())) { throw new Catalog.TableNoPermissionException(identifier); } @@ -1287,25 +1302,29 @@ private MockResponse functionsApiHandler( .collect(Collectors.toList()); return generateFinalListFunctionsResponse(parameters, functions); case "POST": - CreateFunctionRequest requestBody = - RESTApi.fromJson(data, CreateFunctionRequest.class); + CreateFunctionRequest requestBody = parseRequest(data, CreateFunctionRequest.class); String functionName = requestBody.name(); Identifier identity = Identifier.create(databaseName, functionName); - if (!functionStore.containsKey(identity.getFullName())) { - Function function = - new FunctionImpl( - identity, - requestBody.inputParams(), - requestBody.returnParams(), - requestBody.isDeterministic(), - requestBody.definitions(), - requestBody.comment(), - requestBody.options()); - functionStore.put(identity.getFullName(), function); - return new MockResponse().setResponseCode(200); - } else { - throw new Catalog.FunctionAlreadyExistException( - Identifier.create(databaseName, functionName)); + synchronized (databaseLifecycleLock(databaseName)) { + if (!databaseStore.containsKey(databaseName)) { + throw new Catalog.DatabaseNotExistException(databaseName); + } + if (!functionStore.containsKey(identity.getFullName())) { + Function function = + new FunctionImpl( + identity, + requestBody.inputParams(), + requestBody.returnParams(), + requestBody.isDeterministic(), + requestBody.definitions(), + requestBody.comment(), + requestBody.options()); + functionStore.put(identity.getFullName(), function); + return new MockResponse().setResponseCode(200); + } else { + throw new Catalog.FunctionAlreadyExistException( + Identifier.create(databaseName, functionName)); + } } default: return new MockResponse().setResponseCode(404); @@ -1315,6 +1334,17 @@ private MockResponse functionsApiHandler( private MockResponse functionApiHandler( Identifier identifier, String method, String data, Map parameters) throws Exception { + synchronized (databaseLifecycleLock(identifier.getDatabaseName())) { + if (!databaseStore.containsKey(identifier.getDatabaseName())) { + throw new Catalog.DatabaseNotExistException(identifier.getDatabaseName()); + } + return functionApiHandlerInDatabase(identifier, method, data, parameters); + } + } + + private MockResponse functionApiHandlerInDatabase( + Identifier identifier, String method, String data, Map parameters) + throws Exception { if (!functionStore.containsKey(identifier.getFullName())) { throw new Catalog.FunctionNotExistException(identifier); } @@ -1332,8 +1362,7 @@ private MockResponse functionApiHandler( GetFunctionResponse response = toGetFunctionResponse(function); return mockResponse(response, 200); case "POST": - AlterFunctionRequest requestBody = - RESTApi.fromJson(data, AlterFunctionRequest.class); + AlterFunctionRequest requestBody = parseRequest(data, AlterFunctionRequest.class); HashMap newDefinitions = new HashMap<>(function.definitions()); Map newOptions = @@ -1485,15 +1514,17 @@ private MockResponse databasesApiHandler( .collect(Collectors.toList()); return generateFinalListDatabasesResponse(parameters, databases); case "POST": - CreateDatabaseRequest requestBody = - RESTApi.fromJson(data, CreateDatabaseRequest.class); + CreateDatabaseRequest requestBody = parseRequest(data, CreateDatabaseRequest.class); String databaseName = requestBody.getName(); if (noPermissionDatabases.contains(databaseName)) { throw new Catalog.DatabaseNoPermissionException(databaseName); } - catalog.createDatabase(databaseName, false); - databaseStore.put( - databaseName, Database.of(databaseName, requestBody.getOptions(), null)); + synchronized (databaseLifecycleLock(databaseName)) { + catalog.createDatabase(databaseName, false); + databaseStore.put( + databaseName, + Database.of(databaseName, requestBody.getOptions(), null)); + } return new MockResponse().setResponseCode(200); default: return new MockResponse().setResponseCode(404); @@ -1598,9 +1629,12 @@ private int compareTo(Object o1, Object o2) { private MockResponse databaseHandle(String method, String data, String databaseName) throws Exception { - RESTResponse response; - Database database; - if (databaseStore.containsKey(databaseName)) { + synchronized (databaseLifecycleLock(databaseName)) { + RESTResponse response; + Database database; + if (!databaseStore.containsKey(databaseName)) { + return new MockResponse().setResponseCode(404); + } switch (method) { case "GET": database = databaseStore.get(databaseName); @@ -1621,6 +1655,8 @@ private MockResponse databaseHandle(String method, String data, String databaseN removeDatabaseTableState(databaseName); permissionStore.executeAtomically( () -> { + removeDatabaseObjects(viewStore, databaseName); + removeDatabaseObjects(functionStore, databaseName); databaseStore.remove(databaseName); permissionStore.removeDatabase(databaseName); return null; @@ -1628,7 +1664,7 @@ private MockResponse databaseHandle(String method, String data, String databaseN return new MockResponse().setResponseCode(200); case "POST": AlterDatabaseRequest requestBody = - RESTApi.fromJson(data, AlterDatabaseRequest.class); + parseRequest(data, AlterDatabaseRequest.class); List changes = new ArrayList<>(); for (String property : requestBody.getRemovals()) { changes.add(PropertyChange.removeProperty(property)); @@ -1665,7 +1701,6 @@ private MockResponse databaseHandle(String method, String data, String databaseN return new MockResponse().setResponseCode(404); } } - return new MockResponse().setResponseCode(404); } private void removeDatabaseTableState(String databaseName) { @@ -1693,6 +1728,22 @@ private void removeDatabaseTableState(String databaseName) { } } + private Object databaseLifecycleLock(String databaseName) { + return databaseLifecycleLocks.computeIfAbsent(databaseName, ignored -> new Object()); + } + + private Object[] orderedDatabaseLifecycleLocks(String left, String right) { + return left.compareTo(right) <= 0 + ? new Object[] {databaseLifecycleLock(left), databaseLifecycleLock(right)} + : new Object[] {databaseLifecycleLock(right), databaseLifecycleLock(left)}; + } + + private static void removeDatabaseObjects(Map objects, String databaseName) { + objects.keySet() + .removeIf( + name -> databaseName.equals(Identifier.fromString(name).getDatabaseName())); + } + private MockResponse tablesHandle( String method, String data, String databaseName, Map parameters) throws Exception { @@ -1702,31 +1753,38 @@ private MockResponse tablesHandle( List tables = listTables(databaseName, parameters); return generateFinalListTablesResponse(parameters, tables); case "POST": - CreateTableRequest requestBody = - RESTApi.fromJson(data, CreateTableRequest.class); + CreateTableRequest requestBody = parseRequest(data, CreateTableRequest.class); Identifier identifier = requestBody.getIdentifier(); - synchronized (tableLifecycleLocks.lock(identifier.getFullName())) { - if (tableMetadataStore.containsKey(identifier.getFullName())) { - throw new Catalog.TableAlreadyExistException(identifier); + checkArgument( + databaseName.equals(identifier.getDatabaseName()), + "The database in the table identifier must match the request path."); + synchronized (databaseLifecycleLock(databaseName)) { + if (!databaseStore.containsKey(databaseName)) { + throw new Catalog.DatabaseNotExistException(databaseName); } - Schema schema = requestBody.getSchema(); - TableMetadata tableMetadata; - if (isObjectTable(schema)) { - tableMetadata = createObjectTable(identifier, schema); - } else { - catalog.createTable(identifier, schema, false); - boolean isExternal = - schema.options() != null - && schema.options().containsKey(PATH.key()); - tableMetadata = - createTableMetadata( - requestBody.getIdentifier(), - 0L, - requestBody.getSchema(), - UUID.randomUUID().toString(), - isExternal); + synchronized (tableLifecycleLocks.lock(identifier.getFullName())) { + if (tableMetadataStore.containsKey(identifier.getFullName())) { + throw new Catalog.TableAlreadyExistException(identifier); + } + Schema schema = requestBody.getSchema(); + TableMetadata tableMetadata; + if (isObjectTable(schema)) { + tableMetadata = createObjectTable(identifier, schema); + } else { + catalog.createTable(identifier, schema, false); + boolean isExternal = + schema.options() != null + && schema.options().containsKey(PATH.key()); + tableMetadata = + createTableMetadata( + requestBody.getIdentifier(), + 0L, + requestBody.getSchema(), + UUID.randomUUID().toString(), + isExternal); + } + tableMetadataStore.put(identifier.getFullName(), tableMetadata); } - tableMetadataStore.put(identifier.getFullName(), tableMetadata); } return new MockResponse().setResponseCode(200); default: @@ -1944,9 +2002,7 @@ private MockResponse tableHandle(String method, String data, Identifier identifi TableMetadata tableMetadata; if (identifier.isSystemTable()) { TableSchema schema = catalog.loadTableSchema(identifier); - tableMetadata = - createTableMetadata( - identifier, schema.id(), schema.toSchema(), null, false); + tableMetadata = createTableMetadata(identifier, schema, null, false); } else { tableMetadata = tableMetadataStore.get(identifier.getFullName()); } @@ -1968,7 +2024,7 @@ private MockResponse tableHandle(String method, String data, Identifier identifi "updated"); return mockResponse(response, 200); case "POST": - AlterTableRequest requestBody = RESTApi.fromJson(data, AlterTableRequest.class); + AlterTableRequest requestBody = parseRequest(data, AlterTableRequest.class); alterTableImpl(identifier, requestBody.getChanges()); return new MockResponse().setResponseCode(200); case "DELETE": @@ -2003,7 +2059,7 @@ private MockResponse tableHandle(String method, String data, Identifier identifi } private MockResponse replaceTableHandle(Identifier identifier, String data) throws Exception { - ReplaceTableRequest requestBody = RESTApi.fromJson(data, ReplaceTableRequest.class); + ReplaceTableRequest requestBody = parseRequest(data, ReplaceTableRequest.class); Schema newSchema = requestBody.getSchema(); TableMetadata tableMetadata = tableMetadataStore.get(identifier.getFullName()); if (tableMetadata == null) { @@ -2022,8 +2078,9 @@ private MockResponse replaceTableHandle(Identifier identifier, String data) thro current.uuid(), current.isExternal()) .schema(); - validatePoliciesForSchema(identifier, current.uuid(), replacementSchema); - validatePermissionsForSchema(identifier, current.schema(), replacementSchema); + validatePoliciesForSchema(identifier, current.uuid(), replacementSchema, policyStore); + validatePermissionsForSchema( + identifier, current.schema(), replacementSchema, permissionStore); if (isFormatTable(current.schema().toSchema()) || isFormatTable(newSchema)) { throw new UnsupportedOperationException( "replaceTable does not support format tables."); @@ -2033,11 +2090,7 @@ private MockResponse replaceTableHandle(Identifier identifier, String data) thro permissionStore.evolveTableColumns(identifier, current.schema(), replacedSchema); TableMetadata newTableMetadata = createTableMetadata( - identifier, - replacedSchema.id(), - replacedSchema.toSchema(), - current.uuid(), - current.isExternal()); + identifier, replacedSchema, current.uuid(), current.isExternal()); tableMetadataStore.put(identifier.getFullName(), newTableMetadata); FileStoreTable table = (FileStoreTable) catalog.getTable(identifier); Snapshot truncateSnapshot = table.snapshotManager().latestSnapshot(); @@ -2054,41 +2107,54 @@ private MockResponse replaceTableHandle(Identifier identifier, String data) thro } private MockResponse renameTableHandle(String data) throws Exception { - RenameTableRequest requestBody = RESTApi.fromJson(data, RenameTableRequest.class); + RenameTableRequest requestBody = parseRequest(data, RenameTableRequest.class); Identifier fromTable = requestBody.getSource(); Identifier toTable = requestBody.getDestination(); - if (noPermissionTables.contains(fromTable.getFullName())) { - throw new Catalog.TableNoPermissionException(fromTable); - } - Object[] locks = - tableLifecycleLocks.ordered(fromTable.getFullName(), toTable.getFullName()); - synchronized (locks[0]) { - synchronized (locks[1]) { - TableMetadata observed = tableMetadataStore.get(fromTable.getFullName()); - if (observed == null) { - throw new Catalog.TableNotExistException(fromTable); + Object[] databaseLocks = + orderedDatabaseLifecycleLocks( + fromTable.getDatabaseName(), toTable.getDatabaseName()); + synchronized (databaseLocks[0]) { + synchronized (databaseLocks[1]) { + if (!databaseStore.containsKey(fromTable.getDatabaseName())) { + throw new Catalog.DatabaseNotExistException(fromTable.getDatabaseName()); } - synchronized (policyLock(observed.uuid())) { - TableMetadata current = tableMetadataStore.get(fromTable.getFullName()); - if (current == null || !current.uuid().equals(observed.uuid())) { - throw new Catalog.TableNotExistException(fromTable); - } - if (tableMetadataStore.containsKey(toTable.getFullName())) { - throw new Catalog.TableAlreadyExistException(toTable); - } - if (!isFormatTable(current.schema().toSchema()) && !current.isExternal()) { - catalog.renameTable(fromTable, toTable, false); + if (!databaseStore.containsKey(toTable.getDatabaseName())) { + throw new Catalog.DatabaseNotExistException(toTable.getDatabaseName()); + } + if (noPermissionTables.contains(fromTable.getFullName())) { + throw new Catalog.TableNoPermissionException(fromTable); + } + Object[] tableLocks = + tableLifecycleLocks.ordered(fromTable.getFullName(), toTable.getFullName()); + synchronized (tableLocks[0]) { + synchronized (tableLocks[1]) { + TableMetadata observed = tableMetadataStore.get(fromTable.getFullName()); + if (observed == null) { + throw new Catalog.TableNotExistException(fromTable); + } + synchronized (policyLock(observed.uuid())) { + TableMetadata current = tableMetadataStore.get(fromTable.getFullName()); + if (current == null || !current.uuid().equals(observed.uuid())) { + throw new Catalog.TableNotExistException(fromTable); + } + if (tableMetadataStore.containsKey(toTable.getFullName())) { + throw new Catalog.TableAlreadyExistException(toTable); + } + if (!isFormatTable(current.schema().toSchema()) + && !current.isExternal()) { + catalog.renameTable(fromTable, toTable, false); + } + TableMetadata renamedMetadata = + createTableMetadata( + toTable, + current.schema(), + current.uuid(), + current.isExternal()); + tableMetadataStore.remove(fromTable.getFullName(), current); + tableMetadataStore.put(toTable.getFullName(), renamedMetadata); + permissionStore.renameTable(fromTable, toTable); + } } - TableMetadata renamedMetadata = - createTableMetadata( - toTable, - current.schema().id(), - current.schema().toSchema(), - current.uuid(), - current.isExternal()); - tableMetadataStore.remove(fromTable.getFullName(), current); - tableMetadataStore.put(toTable.getFullName(), renamedMetadata); - permissionStore.renameTable(fromTable, toTable); } } } @@ -2118,8 +2184,7 @@ private MockResponse partitionsApiHandle( } return generateFinalListPartitionsResponse(parameters, partitions); case "POST": - CreatePartitionsRequest request = - RESTApi.fromJson(data, CreatePartitionsRequest.class); + CreatePartitionsRequest request = parseRequest(data, CreatePartitionsRequest.class); List storedPartitions = tablePartitionsStore.computeIfAbsent( tableIdentifier.getFullName(), ignored -> new ArrayList<>()); @@ -2258,7 +2323,7 @@ private static long combineLastFileCreationTime( private MockResponse dropPartitionsHandle(String data, Identifier tableIdentifier) throws Exception { - DropPartitionsRequest request = RESTApi.fromJson(data, DropPartitionsRequest.class); + DropPartitionsRequest request = parseRequest(data, DropPartitionsRequest.class); List storedPartitions = tablePartitionsStore.computeIfAbsent( tableIdentifier.getFullName(), ignored -> new ArrayList<>()); @@ -2432,7 +2497,7 @@ private MockResponse branchApiHandle( } } else { CreateBranchRequest requestBody = - RESTApi.fromJson(data, CreateBranchRequest.class); + parseRequest(data, CreateBranchRequest.class); branch = requestBody.branch(); if (requestBody.fromTag() == null) { branchManager.createBranch(requestBody.branch()); @@ -2580,7 +2645,7 @@ private MockResponse tagApiHandle( return new MockResponse().setResponseCode(200); case "POST": // POST /v1/{prefix}/databases/{database}/tables/{table}/tags - CreateTagRequest requestBody = RESTApi.fromJson(data, CreateTagRequest.class); + CreateTagRequest requestBody = parseRequest(data, CreateTagRequest.class); tagName = requestBody.tagName(); Snapshot snapshot; @@ -2684,21 +2749,29 @@ private MockResponse viewsHandle( List views = listViews(databaseName, parameters); return generateFinalListViewsResponse(parameters, views); case "POST": - CreateViewRequest requestBody = RESTApi.fromJson(data, CreateViewRequest.class); + CreateViewRequest requestBody = parseRequest(data, CreateViewRequest.class); Identifier identifier = requestBody.getIdentifier(); + checkArgument( + databaseName.equals(identifier.getDatabaseName()), + "The database in the view identifier must match the request path."); ViewSchema schema = requestBody.getSchema(); - ViewImpl view = - new ViewImpl( - requestBody.getIdentifier(), - schema.fields(), - schema.query(), - schema.dialects(), - schema.comment(), - schema.options()); - if (viewStore.containsKey(identifier.getFullName())) { - throw new Catalog.ViewAlreadyExistException(identifier); + synchronized (databaseLifecycleLock(databaseName)) { + if (!databaseStore.containsKey(databaseName)) { + throw new Catalog.DatabaseNotExistException(databaseName); + } + ViewImpl view = + new ViewImpl( + requestBody.getIdentifier(), + schema.fields(), + schema.query(), + schema.dialects(), + schema.comment(), + schema.options()); + if (viewStore.containsKey(identifier.getFullName())) { + throw new Catalog.ViewAlreadyExistException(identifier); + } + viewStore.put(identifier.getFullName(), view); } - viewStore.put(identifier.getFullName(), view); return new MockResponse().setResponseCode(200); default: return new MockResponse().setResponseCode(404); @@ -2882,6 +2955,16 @@ private List listViews(Map parameters) { private MockResponse viewHandle(String method, Identifier identifier, String requestData) throws Exception { + synchronized (databaseLifecycleLock(identifier.getDatabaseName())) { + if (!databaseStore.containsKey(identifier.getDatabaseName())) { + throw new Catalog.DatabaseNotExistException(identifier.getDatabaseName()); + } + return viewHandleInDatabase(method, identifier, requestData); + } + } + + private MockResponse viewHandleInDatabase( + String method, Identifier identifier, String requestData) throws Exception { RESTResponse response; if (noPermissionViews.contains(identifier.getFullName())) { throw new Catalog.ViewNoPermissionException(identifier); @@ -2922,7 +3005,7 @@ private MockResponse viewHandle(String method, Identifier identifier, String req case "POST": if (viewStore.containsKey(identifier.getFullName())) { AlterViewRequest request = - RESTApi.fromJson(requestData, AlterViewRequest.class); + parseRequest(requestData, AlterViewRequest.class); ViewImpl view = (ViewImpl) viewStore.get(identifier.getFullName()); HashMap newDialects = new HashMap<>(view.dialects()); Map newOptions = new HashMap<>(view.options()); @@ -2992,28 +3075,40 @@ private MockResponse viewHandle(String method, Identifier identifier, String req } private MockResponse renameViewHandle(String data) throws Exception { - RenameTableRequest requestBody = RESTApi.fromJson(data, RenameTableRequest.class); + RenameTableRequest requestBody = parseRequest(data, RenameTableRequest.class); Identifier fromView = requestBody.getSource(); Identifier toView = requestBody.getDestination(); - if (noPermissionViews.contains(fromView.getFullName())) { - throw new Catalog.ViewNoPermissionException(fromView); - } - if (!viewStore.containsKey(fromView.getFullName())) { - throw new Catalog.ViewNotExistException(fromView); - } - if (viewStore.containsKey(toView.getFullName())) { - throw new Catalog.ViewAlreadyExistException(toView); + Object[] databaseLocks = + orderedDatabaseLifecycleLocks(fromView.getDatabaseName(), toView.getDatabaseName()); + synchronized (databaseLocks[0]) { + synchronized (databaseLocks[1]) { + if (!databaseStore.containsKey(fromView.getDatabaseName())) { + throw new Catalog.DatabaseNotExistException(fromView.getDatabaseName()); + } + if (!databaseStore.containsKey(toView.getDatabaseName())) { + throw new Catalog.DatabaseNotExistException(toView.getDatabaseName()); + } + if (noPermissionViews.contains(fromView.getFullName())) { + throw new Catalog.ViewNoPermissionException(fromView); + } + if (!viewStore.containsKey(fromView.getFullName())) { + throw new Catalog.ViewNotExistException(fromView); + } + if (viewStore.containsKey(toView.getFullName())) { + throw new Catalog.ViewAlreadyExistException(toView); + } + permissionStore.executeAtomically( + () -> { + if (viewStore.containsKey(fromView.getFullName())) { + View view = viewStore.get(fromView.getFullName()); + viewStore.remove(fromView.getFullName()); + viewStore.put(toView.getFullName(), view); + permissionStore.renameView(fromView, toView); + } + return null; + }); + } } - permissionStore.executeAtomically( - () -> { - if (viewStore.containsKey(fromView.getFullName())) { - View view = viewStore.get(fromView.getFullName()); - viewStore.remove(fromView.getFullName()); - viewStore.put(toView.getFullName(), view); - permissionStore.renameView(fromView, toView); - } - return null; - }); return new MockResponse().setResponseCode(200); } @@ -3035,14 +3130,15 @@ protected void alterTableImpl(Identifier identifier, List changes) changes, new LazyField<>(() -> false), new LazyField<>(() -> identifier)); - validatePoliciesForSchema(identifier, current.uuid(), candidateSchema); - validatePermissionsForSchema(identifier, current.schema(), candidateSchema); + validatePoliciesForSchema( + identifier, current.uuid(), candidateSchema, policyStore); + validatePermissionsForSchema( + identifier, current.schema(), candidateSchema, permissionStore); if (isFormatTable(schema.toSchema())) { TableMetadata newTableMetadata = createTableMetadata( identifier, - candidateSchema.id(), - candidateSchema.toSchema(), + candidateSchema, current.uuid(), current.isExternal()); tableMetadataStore.put(identifier.getFullName(), newTableMetadata); @@ -3056,11 +3152,7 @@ protected void alterTableImpl(Identifier identifier, List changes) permissionStore.evolveTableColumns(identifier, current.schema(), newSchema); TableMetadata newTableMetadata = createTableMetadata( - identifier, - newSchema.id(), - newSchema.toSchema(), - current.uuid(), - current.isExternal()); + identifier, newSchema, current.uuid(), current.isExternal()); tableMetadataStore.put(identifier.getFullName(), newTableMetadata); } catch (Catalog.TableNotExistException | Catalog.ColumnAlreadyExistException @@ -3271,6 +3363,12 @@ private MockResponse mockResponse(RESTResponse response, int httpCode) { private TableMetadata createTableMetadata( Identifier identifier, long schemaId, Schema schema, String uuid, boolean isExternal) { + return createTableMetadata( + identifier, TableSchema.create(schemaId, schema), uuid, isExternal); + } + + private TableMetadata createTableMetadata( + Identifier identifier, TableSchema schema, String uuid, boolean isExternal) { Map options = new HashMap<>(schema.options()); Path path = isExternal && Objects.nonNull(schema.options().get(PATH.key())) @@ -3286,16 +3384,7 @@ private TableMetadata createTableMetadata( .replaceFirst(LocalFileIOLoader.SCHEME, RESTFileIOTestLoader.SCHEME); } options.put(PATH.key(), restPath); - TableSchema tableSchema = - new TableSchema( - schemaId, - schema.fields(), - schema.fields().size() - 1, - schema.partitionKeys(), - schema.primaryKeys(), - options, - schema.comment()); - return new TableMetadata(tableSchema, isExternal, uuid); + return new TableMetadata(schema.copy(options), isExternal, uuid); } private TableMetadata createObjectTable(Identifier identifier, Schema schema) { @@ -3366,7 +3455,7 @@ private MockResponse permissionsApiHandler( if ("POST".equals(method) && (permissionUri + "/grant").equals(resourcePath)) { PermissionAssignment assignment = - RESTApi.fromJson(data, GrantPermissionRequest.class).assignment(); + parseRequest(data, GrantPermissionRequest.class).assignment(); MockResponse authorization = validateManagementPermission(assignment.getResource()); if (authorization != null) { return authorization; @@ -3390,7 +3479,7 @@ private MockResponse permissionsApiHandler( } if ("POST".equals(method) && (permissionUri + "/revoke").equals(resourcePath)) { - RevokePermissionRequest request = RESTApi.fromJson(data, RevokePermissionRequest.class); + RevokePermissionRequest request = parseRequest(data, RevokePermissionRequest.class); MockResponse authorization = validateManagementPermission(request.getResource()); if (authorization != null) { return authorization; @@ -3549,96 +3638,9 @@ private MockResponse validatePrincipal(String principal) { 404); } - private static String resourceName(PermissionResource resource) { - switch (resource.getType()) { - case CATALOG: - case CATALOG_ALL: - return "catalog"; - case DATABASE: - case DATABASE_ALL: - return resource.getDatabase(); - case TABLE: - case COLUMN: - return resource.getDatabase() + "." + resource.getTable(); - case FUNCTION: - return resource.getDatabase() + "." + resource.getFunction(); - case VIEW: - return resource.getDatabase() + "." + resource.getView(); - default: - return resource.getType().name(); - } - } - - private static int getPermissionMaxResults(Map parameters) { - String strMaxResults = parameters.get(MAX_RESULTS); - if (strMaxResults == null) { - return DEFAULT_MAX_RESULTS; - } - int maxResults = Integer.parseInt(strMaxResults); - return Math.max(1, Math.min(maxResults, ListPermissionsRequest.MAX_PAGE_SIZE)); - } - - private static PagedList buildManagementPage( - List elements, - int maxResults, - @Nullable String pageToken, - java.util.function.Function sortKey) { - String after = decodeManagementPageToken(pageToken); - List remaining = - elements.stream() - .sorted(Comparator.comparing(sortKey)) - .filter( - element -> - after == null - || sortKey.apply(element).compareTo(after) > 0) - .collect(Collectors.toList()); - int end = Math.min(maxResults, remaining.size()); - List page = new ArrayList<>(remaining.subList(0, end)); - String nextPageToken = - end < remaining.size() - ? encodeManagementPageToken(sortKey.apply(page.get(page.size() - 1))) - : null; - return new PagedList<>(page, nextPageToken); - } - - @Nullable - private static String decodeManagementPageToken(@Nullable String pageToken) { - if (pageToken == null) { - return null; - } - String decoded; - try { - decoded = new String(Base64.getUrlDecoder().decode(pageToken), StandardCharsets.UTF_8); - } catch (IllegalArgumentException e) { - throw new IllegalArgumentException("Invalid management page token.", e); - } - checkArgument(decoded.startsWith("v1\0"), "Invalid management page token version."); - return decoded.substring(3); - } - - private static String encodeManagementPageToken(String sortKey) { - return Base64.getUrlEncoder() - .withoutPadding() - .encodeToString(("v1\0" + sortKey).getBytes(StandardCharsets.UTF_8)); - } - - private static boolean matches( - Map parameters, String key, @Nullable String value) { - return !parameters.containsKey(key) || Objects.equals(parameters.get(key), value); - } - - private static PermissionResource permissionResource(Map parameters) { - return new PermissionResource( - ResourceType.fromString(parameters.get("resourceType")), - parameters.get("database"), - parameters.get("table"), - parameters.get("function"), - parameters.get("view")); - } - private boolean isPolicyPath(String resourcePath) { try { - policyPath(resourcePath); + policyPath(resourcePath, permissionUri); return true; } catch (IllegalArgumentException e) { return false; @@ -3648,7 +3650,7 @@ private boolean isPolicyPath(String resourcePath) { private MockResponse policiesApiHandler( String method, String resourcePath, String data, Map parameters) throws JsonProcessingException { - PolicyPath path = policyPath(resourcePath); + PolicyPath path = policyPath(resourcePath, permissionUri); MockResponse authorization = validateManagementPermission(path.resource); if (authorization != null) { return authorization; @@ -3685,7 +3687,7 @@ private MockResponse policiesApiHandler( } if ("POST".equals(method) && !path.drop) { - DataPolicy policy = RESTApi.fromJson(data, PolicyRequest.class).policy(path.resource); + DataPolicy policy = parseRequest(data, PolicyRequest.class).policy(path.resource); String resourceName = policyResourceName(policy); synchronized (policyLock(tableUuid)) { MockResponse targetError = validatePolicyTableVersion(path.resource, tableUuid); @@ -3696,7 +3698,7 @@ private MockResponse policiesApiHandler( if (validation != null) { return validation; } - policy = canonicalizePolicy(policy); + policy = canonicalizePolicy(policy, tableMetadata(path.resource).schema()); PolicyKey key = new PolicyKey(tableUuid, policy); if (policyStore.putIfAbsent(key, policy) != null) { return mockResponse( @@ -3712,7 +3714,7 @@ private MockResponse policiesApiHandler( } if ("POST".equals(method) && path.drop) { - DropPolicyRequest request = RESTApi.fromJson(data, DropPolicyRequest.class); + DropPolicyRequest request = parseRequest(data, DropPolicyRequest.class); DataPolicy existing; synchronized (policyLock(tableUuid)) { MockResponse targetError = validatePolicyTableVersion(path.resource, tableUuid); @@ -3790,21 +3792,6 @@ private MockResponse validatePolicy(DataPolicy policy) { return null; } - private static String policyResourceName(DataPolicy policy) { - ColumnMask columnMask = policy.getColumnMask(); - return policy.type().name() - + ":" - + policy.getPrincipal() - + (columnMask == null ? "" : ":" + columnMask.getOnColumn()); - } - - private static String policyResourceName(DropPolicyRequest request) { - return request.getType().name() - + ":" - + request.getPrincipal() - + (request.getColumn() == null ? "" : ":" + request.getColumn()); - } - @Nullable private TableMetadata tableMetadata(PermissionResource resource) { return tableMetadataStore.get( @@ -3833,175 +3820,12 @@ private MockResponse validatePolicyTableVersion( 409); } - private static DataPolicy withResource(DataPolicy policy, PermissionResource resource) { - return policy.getRowFilter() == null - ? DataPolicy.columnMask(resource, policy.getColumnMask(), policy.getPrincipal()) - : DataPolicy.rowFilter(resource, policy.getRowFilter(), policy.getPrincipal()); - } - - private DataPolicy canonicalizePolicy(DataPolicy policy) { - Identifier identifier = - Identifier.create( - policy.getResource().getDatabase(), policy.getResource().getTable()); - TableSchema schema = tableMetadataStore.get(identifier.getFullName()).schema(); - if (policy.getRowFilter() != null) { - String predicate = - JsonSerdeUtil.toFlatJson(parseRowFilter(schema, policy.getRowFilter())); - return DataPolicy.rowFilter( - policy.getResource(), new RowFilter(predicate), policy.getPrincipal()); - } - ColumnMask columnMask = policy.getColumnMask(); - String transform = JsonSerdeUtil.toFlatJson(parseColumnMask(schema, columnMask)); - return DataPolicy.columnMask( - policy.getResource(), - new ColumnMask(columnMask.getOnColumn(), transform), - policy.getPrincipal()); - } - - private static Predicate parseRowFilter(TableSchema schema, RowFilter rowFilter) { - Predicate predicate = JsonSerdeUtil.fromJson(rowFilter.getPredicate(), Predicate.class); - checkArgument(predicate != null, "Row filter predicate cannot be JSON null."); - Predicate remapped = - TableQueryAuthResult.remapPredicate(predicate, schema.logicalRowType()); - checkArgument(remapped != null, "Row filter predicate cannot be empty."); - return remapped; - } - - private static Transform parseColumnMask(TableSchema schema, ColumnMask columnMask) { - Transform transform = JsonSerdeUtil.fromJson(columnMask.getTransform(), Transform.class); - checkArgument(transform != null, "Column mask transform cannot be JSON null."); - RowType rowType = schema.logicalRowType(); - List remappedInputs = new ArrayList<>(); - for (Object input : transform.inputs()) { - if (input instanceof FieldRef) { - FieldRef ref = (FieldRef) input; - int index = rowType.getFieldIndex(ref.name()); - checkArgument( - index >= 0, - "Column masking refers to field '%s' which is not present in table schema.", - ref.name()); - remappedInputs.add(new FieldRef(index, ref.name(), rowType.getTypeAt(index))); - } else { - remappedInputs.add(input); - } - } - Transform remapped = transform.copyWithNewInputs(remappedInputs); - int targetIndex = rowType.getFieldIndex(columnMask.getOnColumn()); - checkArgument( - targetIndex >= 0, - "Policy column %s does not exist in table schema.", - columnMask.getOnColumn()); - checkArgument( - rowType.getTypeAt(targetIndex).equals(remapped.outputType()), - "Column mask output type %s does not match target column %s type %s.", - remapped.outputType(), - columnMask.getOnColumn(), - rowType.getTypeAt(targetIndex)); - return remapped; - } - private void removePolicies(@Nullable String tableUuid) { if (tableUuid != null) { policyStore.keySet().removeIf(key -> key.tableUuid.equals(tableUuid)); } } - private void validatePoliciesForSchema( - Identifier identifier, @Nullable String tableUuid, TableSchema schema) { - if (tableUuid == null) { - return; - } - List policies = - policyStore.entrySet().stream() - .filter(entry -> entry.getKey().tableUuid.equals(tableUuid)) - .map(Map.Entry::getValue) - .collect(Collectors.toList()); - if (policies.isEmpty()) { - return; - } - checkArgument( - CoreOptions.fromMap(schema.options()).queryAuthEnabled(), - "Cannot disable query-auth.enabled while table %s has data policies.", - identifier.getFullName()); - - Set columns = new HashSet<>(schema.fieldNames()); - for (DataPolicy policy : policies) { - ColumnMask columnMask = policy.getColumnMask(); - if (columnMask != null) { - checkArgument( - columns.contains(columnMask.getOnColumn()), - "Cannot remove or rename policy column %s from table %s.", - columnMask.getOnColumn(), - identifier.getFullName()); - } - if (policy.getRowFilter() == null) { - parseColumnMask(schema, columnMask); - } else { - parseRowFilter(schema, policy.getRowFilter()); - } - } - } - - private void validatePermissionsForSchema( - Identifier identifier, TableSchema previous, TableSchema current) { - if (permissionStore.hasColumnAssignments(identifier)) { - checkArgument( - CoreOptions.fromMap(current.options()).queryAuthEnabled(), - "Cannot disable query-auth.enabled while table %s has column permissions.", - identifier.getFullName()); - checkArgument( - permissionStore.canEvolveTableColumns(identifier, previous, current), - "Cannot drop every allowed column while table %s has column permissions.", - identifier.getFullName()); - } - } - - private static boolean matchesPolicy(DataPolicy policy, Map parameters) { - if (!matches(parameters, "type", policy.type().name())) { - return false; - } - if (parameters.containsKey("column")) { - ColumnMask columnMask = policy.getColumnMask(); - if (columnMask == null - || !Objects.equals(parameters.get("column"), columnMask.getOnColumn())) { - return false; - } - } - return !parameters.containsKey("principal") - || policy.getPrincipal().equals(parameters.get("principal")); - } - - private PolicyPath policyPath(String resourcePath) { - String catalogBase = StringUtils.substringBeforeLast(permissionUri, "/"); - checkArgument(resourcePath.startsWith(catalogBase + "/"), "Not a catalog policy path."); - String[] parts = resourcePath.substring(catalogBase.length() + 1).split("/"); - if ((parts.length == 5 || (parts.length == 6 && "drop".equals(parts[5]))) - && "databases".equals(parts[0]) - && "tables".equals(parts[2]) - && "policies".equals(parts[4])) { - return new PolicyPath( - new PermissionResource( - ResourceType.TABLE, - RESTUtil.decodeString(parts[1]), - RESTUtil.decodeString(parts[3]), - null, - null), - parts.length == 6); - } - throw new IllegalArgumentException("Not a policy path."); - } - - private static class PolicyPath { - - private final PermissionResource resource; - private final boolean drop; - - private PolicyPath(PermissionResource resource, boolean drop) { - this.resource = resource; - this.drop = drop; - } - } - private String getNextPageTokenForEntities(List entities, Integer maxResults) { if (entities == null || entities.isEmpty() diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogServerUtils.java b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogServerUtils.java new file mode 100644 index 000000000000..1052b94b9d43 --- /dev/null +++ b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogServerUtils.java @@ -0,0 +1,380 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.rest; + +import org.apache.paimon.CoreOptions; +import org.apache.paimon.PagedList; +import org.apache.paimon.catalog.Identifier; +import org.apache.paimon.catalog.TableQueryAuthResult; +import org.apache.paimon.management.ColumnMask; +import org.apache.paimon.management.DataPolicy; +import org.apache.paimon.management.ListPermissionsRequest; +import org.apache.paimon.management.PermissionResource; +import org.apache.paimon.management.ResourceType; +import org.apache.paimon.management.RowFilter; +import org.apache.paimon.predicate.FieldRef; +import org.apache.paimon.predicate.Predicate; +import org.apache.paimon.predicate.Transform; +import org.apache.paimon.rest.requests.DropPolicyRequest; +import org.apache.paimon.schema.TableSchema; +import org.apache.paimon.types.RowType; +import org.apache.paimon.utils.JsonSerdeUtil; +import org.apache.paimon.utils.StringUtils; + +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.core.JsonProcessingException; + +import javax.annotation.Nullable; + +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Base64; +import java.util.Comparator; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.function.Function; +import java.util.stream.Collectors; + +import static org.apache.paimon.rest.RESTApi.MAX_RESULTS; +import static org.apache.paimon.utils.Preconditions.checkArgument; + +/** Stateless helpers used by {@link RESTCatalogServer}. */ +final class RESTCatalogServerUtils { + + private RESTCatalogServerUtils() {} + + static T parseRequest(String data, Class requestClass) { + try { + return RESTApi.fromJson(data, requestClass); + } catch (JsonProcessingException e) { + Throwable invalidArgument = findCause(e, IllegalArgumentException.class); + throw new InvalidRequestException( + errorMessage(invalidArgument == null ? e : invalidArgument), e); + } + } + + @Nullable + static Throwable findCause(Throwable throwable, Class causeType) { + Throwable current = throwable; + while (current != null) { + if (causeType.isInstance(current)) { + return current; + } + Throwable cause = current.getCause(); + if (cause == current) { + break; + } + current = cause; + } + return null; + } + + static String errorMessage(Throwable throwable) { + Throwable current = throwable; + while (current != null) { + if (StringUtils.isNotEmpty(current.getMessage())) { + return current.getMessage(); + } + Throwable cause = current.getCause(); + if (cause == current) { + break; + } + current = cause; + } + return throwable.getClass().getSimpleName(); + } + + static PagedList buildManagementPage( + List elements, + int maxResults, + @Nullable String pageToken, + Function sortKey) { + String after = decodeManagementPageToken(pageToken); + List remaining = + elements.stream() + .sorted(Comparator.comparing(sortKey)) + .filter( + element -> + after == null + || sortKey.apply(element).compareTo(after) > 0) + .collect(Collectors.toList()); + int end = Math.min(maxResults, remaining.size()); + List page = new ArrayList<>(remaining.subList(0, end)); + String nextPageToken = + end < remaining.size() + ? encodeManagementPageToken(sortKey.apply(page.get(page.size() - 1))) + : null; + return new PagedList<>(page, nextPageToken); + } + + static PermissionResource permissionResource(Map parameters) { + return new PermissionResource( + ResourceType.fromString(parameters.get("resourceType")), + parameters.get("database"), + parameters.get("table"), + parameters.get("function"), + parameters.get("view")); + } + + static String resourceName(PermissionResource resource) { + switch (resource.getType()) { + case CATALOG: + case CATALOG_ALL: + return "catalog"; + case DATABASE: + case DATABASE_ALL: + return resource.getDatabase(); + case TABLE: + case COLUMN: + return resource.getDatabase() + "." + resource.getTable(); + case FUNCTION: + return resource.getDatabase() + "." + resource.getFunction(); + case VIEW: + return resource.getDatabase() + "." + resource.getView(); + default: + return resource.getType().name(); + } + } + + static int getPermissionMaxResults(Map parameters) { + String strMaxResults = parameters.get(MAX_RESULTS); + if (strMaxResults == null) { + return RESTCatalogServer.DEFAULT_MAX_RESULTS; + } + int maxResults = Integer.parseInt(strMaxResults); + return Math.max(1, Math.min(maxResults, ListPermissionsRequest.MAX_PAGE_SIZE)); + } + + static String policyResourceName(DataPolicy policy) { + ColumnMask columnMask = policy.getColumnMask(); + return policy.type().name() + + ":" + + policy.getPrincipal() + + (columnMask == null ? "" : ":" + columnMask.getOnColumn()); + } + + static String policyResourceName(DropPolicyRequest request) { + return request.getType().name() + + ":" + + request.getPrincipal() + + (request.getColumn() == null ? "" : ":" + request.getColumn()); + } + + static DataPolicy withResource(DataPolicy policy, PermissionResource resource) { + return policy.getRowFilter() == null + ? DataPolicy.columnMask(resource, policy.getColumnMask(), policy.getPrincipal()) + : DataPolicy.rowFilter(resource, policy.getRowFilter(), policy.getPrincipal()); + } + + static DataPolicy canonicalizePolicy(DataPolicy policy, TableSchema schema) { + if (policy.getRowFilter() != null) { + String predicate = + JsonSerdeUtil.toFlatJson(parseRowFilter(schema, policy.getRowFilter())); + return DataPolicy.rowFilter( + policy.getResource(), new RowFilter(predicate), policy.getPrincipal()); + } + ColumnMask columnMask = policy.getColumnMask(); + String transform = JsonSerdeUtil.toFlatJson(parseColumnMask(schema, columnMask)); + return DataPolicy.columnMask( + policy.getResource(), + new ColumnMask(columnMask.getOnColumn(), transform), + policy.getPrincipal()); + } + + static Predicate parseRowFilter(TableSchema schema, RowFilter rowFilter) { + Predicate predicate = JsonSerdeUtil.fromJson(rowFilter.getPredicate(), Predicate.class); + checkArgument(predicate != null, "Row filter predicate cannot be JSON null."); + Predicate remapped = + TableQueryAuthResult.remapPredicate(predicate, schema.logicalRowType()); + checkArgument(remapped != null, "Row filter predicate cannot be empty."); + return remapped; + } + + static Transform parseColumnMask(TableSchema schema, ColumnMask columnMask) { + Transform transform = JsonSerdeUtil.fromJson(columnMask.getTransform(), Transform.class); + checkArgument(transform != null, "Column mask transform cannot be JSON null."); + RowType rowType = schema.logicalRowType(); + List remappedInputs = new ArrayList<>(); + for (Object input : transform.inputs()) { + if (input instanceof FieldRef) { + FieldRef ref = (FieldRef) input; + int index = rowType.getFieldIndex(ref.name()); + checkArgument( + index >= 0, + "Column masking refers to field '%s' which is not present in table schema.", + ref.name()); + remappedInputs.add(new FieldRef(index, ref.name(), rowType.getTypeAt(index))); + } else { + remappedInputs.add(input); + } + } + Transform remapped = transform.copyWithNewInputs(remappedInputs); + int targetIndex = rowType.getFieldIndex(columnMask.getOnColumn()); + checkArgument( + targetIndex >= 0, + "Policy column %s does not exist in table schema.", + columnMask.getOnColumn()); + checkArgument( + rowType.getTypeAt(targetIndex).equals(remapped.outputType()), + "Column mask output type %s does not match target column %s type %s.", + remapped.outputType(), + columnMask.getOnColumn(), + rowType.getTypeAt(targetIndex)); + return remapped; + } + + static void validatePoliciesForSchema( + Identifier identifier, + @Nullable String tableUuid, + TableSchema schema, + Map policyStore) { + if (tableUuid == null) { + return; + } + List policies = + policyStore.entrySet().stream() + .filter(entry -> entry.getKey().tableUuid.equals(tableUuid)) + .map(Map.Entry::getValue) + .collect(Collectors.toList()); + if (policies.isEmpty()) { + return; + } + checkArgument( + CoreOptions.fromMap(schema.options()).queryAuthEnabled(), + "Cannot disable query-auth.enabled while table %s has data policies.", + identifier.getFullName()); + + Set columns = new HashSet<>(schema.fieldNames()); + for (DataPolicy policy : policies) { + ColumnMask columnMask = policy.getColumnMask(); + if (columnMask != null) { + checkArgument( + columns.contains(columnMask.getOnColumn()), + "Cannot remove or rename policy column %s from table %s.", + columnMask.getOnColumn(), + identifier.getFullName()); + } + if (policy.getRowFilter() == null) { + parseColumnMask(schema, columnMask); + } else { + parseRowFilter(schema, policy.getRowFilter()); + } + } + } + + static void validatePermissionsForSchema( + Identifier identifier, + TableSchema previous, + TableSchema current, + RESTPermissionStore permissionStore) { + if (permissionStore.hasColumnAssignments(identifier)) { + checkArgument( + CoreOptions.fromMap(current.options()).queryAuthEnabled(), + "Cannot disable query-auth.enabled while table %s has column permissions.", + identifier.getFullName()); + checkArgument( + permissionStore.canEvolveTableColumns(identifier, previous, current), + "Cannot drop every allowed column while table %s has column permissions.", + identifier.getFullName()); + } + } + + static boolean matchesPolicy(DataPolicy policy, Map parameters) { + if (!matches(parameters, "type", policy.type().name())) { + return false; + } + if (parameters.containsKey("column")) { + ColumnMask columnMask = policy.getColumnMask(); + if (columnMask == null + || !Objects.equals(parameters.get("column"), columnMask.getOnColumn())) { + return false; + } + } + return !parameters.containsKey("principal") + || policy.getPrincipal().equals(parameters.get("principal")); + } + + static PolicyPath policyPath(String resourcePath, String permissionUri) { + String catalogBase = StringUtils.substringBeforeLast(permissionUri, "/"); + checkArgument(resourcePath.startsWith(catalogBase + "/"), "Not a catalog policy path."); + String[] parts = resourcePath.substring(catalogBase.length() + 1).split("/"); + if ((parts.length == 5 || (parts.length == 6 && "drop".equals(parts[5]))) + && "databases".equals(parts[0]) + && "tables".equals(parts[2]) + && "policies".equals(parts[4])) { + return new PolicyPath( + new PermissionResource( + ResourceType.TABLE, + RESTUtil.decodeString(parts[1]), + RESTUtil.decodeString(parts[3]), + null, + null), + parts.length == 6); + } + throw new IllegalArgumentException("Not a policy path."); + } + + @Nullable + private static String decodeManagementPageToken(@Nullable String pageToken) { + if (pageToken == null) { + return null; + } + String decoded; + try { + decoded = new String(Base64.getUrlDecoder().decode(pageToken), StandardCharsets.UTF_8); + } catch (IllegalArgumentException e) { + throw new IllegalArgumentException("Invalid management page token.", e); + } + checkArgument(decoded.startsWith("v1\0"), "Invalid management page token version."); + return decoded.substring(3); + } + + private static String encodeManagementPageToken(String sortKey) { + return Base64.getUrlEncoder() + .withoutPadding() + .encodeToString(("v1\0" + sortKey).getBytes(StandardCharsets.UTF_8)); + } + + private static boolean matches( + Map parameters, String key, @Nullable String value) { + return !parameters.containsKey(key) || Objects.equals(parameters.get(key), value); + } + + static final class InvalidRequestException extends RuntimeException { + + private static final long serialVersionUID = 1L; + + private InvalidRequestException(String message, Throwable cause) { + super(message, cause); + } + } + + static final class PolicyPath { + + final PermissionResource resource; + final boolean drop; + + private PolicyPath(PermissionResource resource, boolean drop) { + this.resource = resource; + this.drop = drop; + } + } +} diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/RESTColumnPermissionSupport.java b/paimon-core/src/test/java/org/apache/paimon/rest/RESTColumnPermissionSupport.java index 39f7d72d3305..a94e842ba75e 100644 --- a/paimon-core/src/test/java/org/apache/paimon/rest/RESTColumnPermissionSupport.java +++ b/paimon-core/src/test/java/org/apache/paimon/rest/RESTColumnPermissionSupport.java @@ -74,9 +74,7 @@ static boolean canSelect( List selected = selectedColumns == null ? metadata.schema().fieldNames() : selectedColumns; for (String column : selected) { - int nestedSeparator = column.indexOf('.'); - String topLevel = nestedSeparator < 0 ? column : column.substring(0, nestedSeparator); - if (!included.contains(topLevel)) { + if (!included.contains(column)) { return false; } } diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/RESTPermissionStoreTest.java b/paimon-core/src/test/java/org/apache/paimon/rest/RESTPermissionStoreTest.java index 3d51004872b9..781a8462315d 100644 --- a/paimon-core/src/test/java/org/apache/paimon/rest/RESTPermissionStoreTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/rest/RESTPermissionStoreTest.java @@ -19,6 +19,7 @@ package org.apache.paimon.rest; import org.apache.paimon.catalog.Identifier; +import org.apache.paimon.catalog.TableMetadata; import org.apache.paimon.management.PermissionAssignment; import org.apache.paimon.management.PermissionColumns; import org.apache.paimon.management.PermissionResource; @@ -160,6 +161,34 @@ void testColumnGrantReplacesTheWholeColumnRangeForTheSameIdentity() { .isEqualTo(Arrays.asList("email")); } + @Test + void testColumnSelectionUsesExactTopLevelNamesContainingDots() { + RESTPermissionStore store = new RESTPermissionStore(); + store.put( + new PermissionAssignment( + columnResource(), + "SELECT", + ANALYST, + new PermissionColumns(Collections.singletonList("public"), null), + null)); + TableMetadata metadata = + new TableMetadata( + tableSchema( + new DataField(0, "public", DataTypes.STRING()), + new DataField(1, "public.secret", DataTypes.STRING())), + false, + "uuid"); + + assertThat( + RESTColumnPermissionSupport.canSelect( + store, + Collections.singleton(ANALYST), + Identifier.create("sales", "orders"), + metadata, + Collections.singletonList("public.secret"))) + .isFalse(); + } + @Test void testTableAndColumnAssignmentsFollowResourceAndSchemaLifecycle() { RESTPermissionStore store = new RESTPermissionStore(); diff --git a/paimon-core/src/test/java/org/apache/paimon/table/source/AbstractDataTableReadTest.java b/paimon-core/src/test/java/org/apache/paimon/table/source/AbstractDataTableReadTest.java index b76c42be202e..667fb324d280 100644 --- a/paimon-core/src/test/java/org/apache/paimon/table/source/AbstractDataTableReadTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/table/source/AbstractDataTableReadTest.java @@ -92,6 +92,10 @@ void testMaskDependenciesPreserveNestedProjectionAndSkipUnselectedMasks() throws assertThat(read.appliedReadType().getFieldNames()) .containsExactly("profile", "protected", "seed"); assertThat(read.appliedReadType().getTypeAt(0)).isEqualTo(prunedProfile); + + read.createAuthedReader(new TableQueryAuthResult(null, Collections.emptyMap())); + + assertThat(read.appliedReadType()).isEqualTo(requestedType); } private static class TestingDataTableRead extends AbstractDataTableRead { From bf2155225ece2e50604157021608558205321238 Mon Sep 17 00:00:00 2001 From: JingsongLi Date: Wed, 26 Aug 2026 20:20:53 +0800 Subject: [PATCH 5/5] [fix] Preserve default read projection across splits --- .../table/source/AbstractDataTableRead.java | 11 +++++++---- .../source/AbstractDataTableReadTest.java | 18 ++++++++++++++++++ 2 files changed, 25 insertions(+), 4 deletions(-) diff --git a/paimon-core/src/main/java/org/apache/paimon/table/source/AbstractDataTableRead.java b/paimon-core/src/main/java/org/apache/paimon/table/source/AbstractDataTableRead.java index 9bf4982552dc..9a744a9af0be 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/source/AbstractDataTableRead.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/source/AbstractDataTableRead.java @@ -123,10 +123,13 @@ protected final QueryAuthContext unwrapQueryAuthSplit(Split split) { protected final RecordReader createDataReader( Split split, @Nullable TableQueryAuthResult authResult) throws IOException { - // A TableRead can be reused for multiple splits. Authentication may have expanded the - // physical projection for the previous split, so always restore the logical projection - // before applying the current split's authorization dependencies. - applyReadType(currentReadType()); + // A TableRead can be reused for multiple splits. Authentication may have expanded an + // explicitly configured physical projection for the previous split, so restore it before + // applying the current split's authorization dependencies. Without an explicit projection, + // the underlying reader must retain its own default read type. + if (readType != null) { + applyReadType(readType); + } RecordReader reader; if (authResult == null) { reader = reader(split); diff --git a/paimon-core/src/test/java/org/apache/paimon/table/source/AbstractDataTableReadTest.java b/paimon-core/src/test/java/org/apache/paimon/table/source/AbstractDataTableReadTest.java index 667fb324d280..fa1934c4fe43 100644 --- a/paimon-core/src/test/java/org/apache/paimon/table/source/AbstractDataTableReadTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/table/source/AbstractDataTableReadTest.java @@ -44,6 +44,24 @@ /** Tests query-authorization projection expansion in {@link AbstractDataTableRead}. */ class AbstractDataTableReadTest { + @Test + void testNoProjectionResetWithoutExplicitReadType() throws IOException { + TableSchema schema = + new TableSchema( + 1, + Collections.singletonList(new DataField(0, "value", DataTypes.STRING())), + 0, + Collections.emptyList(), + Collections.emptyList(), + Collections.emptyMap(), + null); + TestingDataTableRead read = new TestingDataTableRead(schema); + + read.createReader(mock(Split.class)); + + assertThat(read.appliedReadType()).isNull(); + } + @Test void testMaskDependenciesPreserveNestedProjectionAndSkipUnselectedMasks() throws IOException { RowType fullProfile =