From e41f1638ad958cde9183d2502eb0e688a2577569 Mon Sep 17 00:00:00 2001 From: guoqiang Date: Thu, 27 Aug 2026 17:07:51 +0800 Subject: [PATCH 1/2] [fix](fe) Use service-specific Ranger select access types ### What problem does this PR solve? Issue Number: None Related PR: None Problem Summary: RangerAccessController hardcoded the Doris uppercase SELECT access type for row-filter and data-mask requests. Ranger-Hive policies define the access type as lowercase select, so optimized policy evaluation could skip those policies after its exact-match index was initialized. Delegate SELECT mapping to each Ranger service while retaining a concrete uppercase default for compatibility with existing subclasses. Ranger-Hive uses its lowercase mapping consistently. Ranger connection settings remain process-scoped and are loaded from the existing ranger--security.xml configuration. ### Release note Fix Ranger-Hive row filtering and data masking for lowercase select policies. ### Check List (For Author) - Test: Regression test and Unit Test - Unit Test: ./run-fe-ut.sh --run org.apache.doris.catalog.authorizer.ranger.hive.RangerHiveAccessControllerTest - Regression test: ./run-regression-test.sh --run --conf -d ranger_p2 -s test_ranger_hive_lowercase_access_type - Behavior changed: Yes. Ranger-Hive row-filter and data-mask policies now match lowercase select access entries. - Does this need documentation: No --- .../ranger/RangerAccessController.java | 11 +- .../hive/RangerHiveAccessController.java | 18 +- .../hive/RangerHiveAccessControllerTest.java | 62 ++++++ ...test_ranger_hive_lowercase_access_type.out | 7 + ...t_ranger_hive_lowercase_access_type.groovy | 202 ++++++++++++++++++ 5 files changed, 291 insertions(+), 9 deletions(-) create mode 100644 fe/fe-core/src/test/java/org/apache/doris/catalog/authorizer/ranger/hive/RangerHiveAccessControllerTest.java create mode 100644 regression-test/data/ranger_p2/test_ranger_hive_lowercase_access_type.out create mode 100644 regression-test/suites/ranger_p2/test_ranger_hive_lowercase_access_type.groovy diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/authorizer/ranger/RangerAccessController.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/authorizer/ranger/RangerAccessController.java index 7a2779b43b1c6b..eda20d58d3701a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/authorizer/ranger/RangerAccessController.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/authorizer/ranger/RangerAccessController.java @@ -95,9 +95,8 @@ public List evalRowFilterPolicies(UserIdentity curren RangerAccessRequestImpl request = createRequest(currentUser); // If the access type is not set here, it defaults to ANY1 ACCESS. // The internal logic of the ranger is to traverse all permission items. - // Since the ranger UI will set the access type to 'SELECT', - // we will keep it consistent with the UI here to avoid performance issues - request.setAccessType(DorisAccessType.SELECT.name()); + // Keep the access type consistent with the service definition to avoid traversing all policy items. + request.setAccessType(getSelectAccessType()); request.setResource(resource); if (LOG.isDebugEnabled()) { @@ -125,7 +124,7 @@ public Optional evalDataMaskPolicy(UserIdentity currentUser, Str String col) { RangerAccessResourceImpl resource = createResource(ctl, db, tbl, col); RangerAccessRequestImpl request = createRequest(currentUser); - request.setAccessType(DorisAccessType.SELECT.name()); + request.setAccessType(getSelectAccessType()); request.setResource(resource); if (LOG.isDebugEnabled()) { @@ -167,6 +166,10 @@ public Optional evalDataMaskPolicy(UserIdentity currentUser, Str protected abstract RangerAccessRequestImpl createRequest(UserIdentity currentUser); + protected String getSelectAccessType() { + return DorisAccessType.SELECT.name(); + } + protected abstract RangerAccessResourceImpl createResource(String ctl, String db, String tbl); protected abstract RangerAccessResourceImpl createResource(String ctl, String db, String tbl, String col); diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/authorizer/ranger/hive/RangerHiveAccessController.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/authorizer/ranger/hive/RangerHiveAccessController.java index 6f862ceade1f1f..0297f88e6e71dd 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/authorizer/ranger/hive/RangerHiveAccessController.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/authorizer/ranger/hive/RangerHiveAccessController.java @@ -43,6 +43,7 @@ import java.util.Collection; import java.util.Date; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Optional; import java.util.Set; @@ -113,14 +114,16 @@ public void close() { private RangerAccessRequestImpl createRequest(UserIdentity currentUser, HiveAccessType accessType) { RangerAccessRequestImpl request = createRequest(currentUser); - if (accessType == HiveAccessType.USE) { - request.setAccessType(RangerPolicyEngine.ANY_ACCESS); - } else { - request.setAccessType(accessType.name().toLowerCase()); - } + request.setAccessType(toRangerAccessType(accessType)); return request; } + static String toRangerAccessType(HiveAccessType accessType) { + return accessType == HiveAccessType.USE + ? RangerPolicyEngine.ANY_ACCESS + : accessType.name().toLowerCase(Locale.ROOT); + } + @Override protected RangerAccessRequestImpl createRequest(UserIdentity currentUser) { RangerAccessRequestImpl request = new RangerAccessRequestImpl(); @@ -137,6 +140,11 @@ protected RangerAccessRequestImpl createRequest(UserIdentity currentUser) { return request; } + @Override + protected String getSelectAccessType() { + return toRangerAccessType(HiveAccessType.SELECT); + } + private void checkPrivileges(UserIdentity currentUser, HiveAccessType accessType, List hiveResources) throws AuthorizationException { lifecycleLock.readLock().lock(); diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/authorizer/ranger/hive/RangerHiveAccessControllerTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/authorizer/ranger/hive/RangerHiveAccessControllerTest.java new file mode 100644 index 00000000000000..95c3cc2a690c01 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/authorizer/ranger/hive/RangerHiveAccessControllerTest.java @@ -0,0 +1,62 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.catalog.authorizer.ranger.hive; + +import org.apache.doris.analysis.UserIdentity; + +import org.apache.ranger.plugin.policyengine.RangerAccessRequestImpl; +import org.apache.ranger.plugin.policyengine.RangerPolicyEngine; +import org.apache.ranger.plugin.service.RangerBasePlugin; +import org.junit.Assert; +import org.junit.Test; +import org.mockito.Mockito; + +import java.lang.reflect.Field; +import java.util.concurrent.locks.ReentrantReadWriteLock; + +public class RangerHiveAccessControllerTest { + @Test + public void testRangerAccessTypeMapping() { + Assert.assertEquals("select", RangerHiveAccessController.toRangerAccessType(HiveAccessType.SELECT)); + Assert.assertEquals("update", RangerHiveAccessController.toRangerAccessType(HiveAccessType.UPDATE)); + Assert.assertEquals(RangerPolicyEngine.ANY_ACCESS, + RangerHiveAccessController.toRangerAccessType(HiveAccessType.USE)); + } + + @Test + public void testPolicyRequestsUseLowerCaseSelect() throws Exception { + RangerHiveAccessController controller = Mockito.mock( + RangerHiveAccessController.class, Mockito.CALLS_REAL_METHODS); + Field lifecycleLock = RangerHiveAccessController.class.getDeclaredField("lifecycleLock"); + lifecycleLock.setAccessible(true); + lifecycleLock.set(controller, new ReentrantReadWriteLock()); + RangerBasePlugin plugin = Mockito.mock(RangerBasePlugin.class); + UserIdentity currentUser = UserIdentity.createAnalyzedUserIdentWithIp("user", "%"); + RangerAccessRequestImpl rowFilterRequest = new RangerAccessRequestImpl(); + RangerAccessRequestImpl dataMaskRequest = new RangerAccessRequestImpl(); + + Mockito.doReturn(rowFilterRequest, dataMaskRequest).when(controller).createRequest(currentUser); + Mockito.doReturn(plugin).when(controller).getPlugin(); + + controller.evalRowFilterPolicies(currentUser, "catalog", "database", "table"); + controller.evalDataMaskPolicy(currentUser, "catalog", "database", "table", "column"); + + Assert.assertEquals("select", rowFilterRequest.getAccessType()); + Assert.assertEquals("select", dataMaskRequest.getAccessType()); + } +} diff --git a/regression-test/data/ranger_p2/test_ranger_hive_lowercase_access_type.out b/regression-test/data/ranger_p2/test_ranger_hive_lowercase_access_type.out new file mode 100644 index 00000000000000..0a48252701f19e --- /dev/null +++ b/regression-test/data/ranger_p2/test_ranger_hive_lowercase_access_type.out @@ -0,0 +1,7 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !lowercase_access_type_first -- +2 \N + +-- !lowercase_access_type_warmed_up -- +2 \N + diff --git a/regression-test/suites/ranger_p2/test_ranger_hive_lowercase_access_type.groovy b/regression-test/suites/ranger_p2/test_ranger_hive_lowercase_access_type.groovy new file mode 100644 index 00000000000000..5639882edf2b28 --- /dev/null +++ b/regression-test/suites/ranger_p2/test_ranger_hive_lowercase_access_type.groovy @@ -0,0 +1,202 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.apache.ranger.RangerClient +import org.apache.ranger.plugin.model.RangerPolicy +import org.apache.ranger.plugin.model.RangerService + + +suite("test_ranger_hive_lowercase_access_type", "p2,ranger,external") { + String enableRangerTest = context.config.otherConfigs.get("enableRangerTest") + String enableHiveTest = context.config.otherConfigs.get("enableHiveTest") + + if (!enableRangerTest?.equalsIgnoreCase("true") || !enableHiveTest?.equalsIgnoreCase("true")) { + logger.info("skip Ranger-Hive lowercase access type case because Ranger or Hive test is not enabled") + return + } + + String rangerEndpoint = context.config.otherConfigs.get("rangerEndpoint") + String rangerUser = context.config.otherConfigs.get("rangerUser") + String rangerPassword = context.config.otherConfigs.get("rangerPassword") + String rangerServiceName = context.config.otherConfigs.get("rangerServiceName") + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + String hmsPort = context.config.otherConfigs.get("hive3HmsPort") + String hiveServerPort = context.config.otherConfigs.get("hive3ServerPort") + String rangerHiveServiceName = "${rangerServiceName}_hive" + + String catalog = "ranger_hive_lowercase_catalog" + String cleanupCatalog = "ranger_hive_lowercase_cleanup_catalog" + String database = "ranger_hive_lowercase_db" + String table = "ranger_hive_lowercase_tbl" + String user = "ranger_hive_lowercase_user" + String password = "C123_567p" + String accessPolicyName = "doris_ranger_hive_lowercase_access" + String rowFilterPolicyName = "doris_ranger_hive_lowercase_row_filter" + String dataMaskPolicyName = "doris_ranger_hive_lowercase_data_mask" + List policyNames = [accessPolicyName, rowFilterPolicyName, dataMaskPolicyName] + + RangerClient rangerClient = new RangerClient( + "http://${rangerEndpoint}", "simple", rangerUser, rangerPassword, null) + boolean createdHiveService = false + + def deletePolicyIfExists = { String policyName -> + try { + rangerClient.deletePolicy(rangerHiveServiceName, policyName) + } catch (Exception e) { + logger.info("Ranger policy ${policyName} does not exist: ${e.getMessage()}") + } + } + + def tableResources = { + Map resources = new HashMap<>() + resources.put("database", new RangerPolicy.RangerPolicyResource(database)) + resources.put("table", new RangerPolicy.RangerPolicyResource(table)) + return resources + } + + def accessResources = { + Map resources = tableResources() + resources.put("column", new RangerPolicy.RangerPolicyResource("*")) + return resources + } + + try { + try { + RangerService hiveService = rangerClient.getService(rangerHiveServiceName) + assertEquals("hive", hiveService.getType()) + } catch (Exception e) { + logger.info("Create Ranger-Hive service ${rangerHiveServiceName}: ${e.getMessage()}") + RangerService hiveService = new RangerService() + hiveService.setType("hive") + hiveService.setName(rangerHiveServiceName) + hiveService.setDisplayName(rangerHiveServiceName) + hiveService.setConfigs([ + "username": "hive", + "password": "hive", + "jdbc.driverClassName": "org.apache.hive.jdbc.HiveDriver", + "jdbc.url": "jdbc:hive2://${externalEnvIp}:${hiveServerPort}".toString() + ]) + rangerClient.createService(hiveService) + createdHiveService = true + } + + // Prepare the Hive data through a catalog without Ranger, so setup and cleanup do not need policies. + sql """DROP CATALOG IF EXISTS `${catalog}`""" + sql """DROP CATALOG IF EXISTS `${cleanupCatalog}`""" + sql """CREATE CATALOG `${catalog}` PROPERTIES ( + 'type' = 'hms', + 'hive.metastore.uris' = 'thrift://${externalEnvIp}:${hmsPort}' + )""" + try_sql """DROP TABLE IF EXISTS `${catalog}`.`${database}`.`${table}`""" + try_sql """DROP DATABASE IF EXISTS `${catalog}`.`${database}`""" + sql """CREATE DATABASE `${catalog}`.`${database}`""" + sql """CREATE TABLE `${catalog}`.`${database}`.`${table}` ( + id BIGINT, + secret VARCHAR(20) + ) ENGINE=hive + PROPERTIES ('file_format' = 'parquet')""" + sql """INSERT INTO `${catalog}`.`${database}`.`${table}` VALUES + (1, 'first'), (2, 'second')""" + sql """DROP CATALOG `${catalog}`""" + + sql """DROP USER IF EXISTS '${user}'""" + sql """CREATE USER '${user}' IDENTIFIED BY '${password}'""" + + policyNames.each { deletePolicyIfExists(it) } + + RangerPolicy accessPolicy = new RangerPolicy() + accessPolicy.setService(rangerHiveServiceName) + accessPolicy.setName(accessPolicyName) + accessPolicy.setResources(accessResources()) + RangerPolicy.RangerPolicyItem accessPolicyItem = new RangerPolicy.RangerPolicyItem() + accessPolicyItem.setUsers([user]) + accessPolicyItem.setAccesses([new RangerPolicy.RangerPolicyItemAccess("select")]) + accessPolicy.setPolicyItems([accessPolicyItem]) + rangerClient.createPolicy(accessPolicy) + + RangerPolicy rowFilterPolicy = new RangerPolicy() + rowFilterPolicy.setService(rangerHiveServiceName) + rowFilterPolicy.setName(rowFilterPolicyName) + rowFilterPolicy.setPolicyType(RangerPolicy.POLICY_TYPE_ROWFILTER) + rowFilterPolicy.setResources(tableResources()) + RangerPolicy.RangerRowFilterPolicyItem rowFilterPolicyItem = new RangerPolicy.RangerRowFilterPolicyItem() + rowFilterPolicyItem.setUsers([user]) + rowFilterPolicyItem.setAccesses([new RangerPolicy.RangerPolicyItemAccess("select")]) + rowFilterPolicyItem.setRowFilterInfo(new RangerPolicy.RangerPolicyItemRowFilterInfo("id >= 2")) + rowFilterPolicy.setRowFilterPolicyItems([rowFilterPolicyItem]) + rangerClient.createPolicy(rowFilterPolicy) + + RangerPolicy dataMaskPolicy = new RangerPolicy() + dataMaskPolicy.setService(rangerHiveServiceName) + dataMaskPolicy.setName(dataMaskPolicyName) + dataMaskPolicy.setPolicyType(RangerPolicy.POLICY_TYPE_DATAMASK) + Map dataMaskResources = tableResources() + dataMaskResources.put("column", new RangerPolicy.RangerPolicyResource("secret")) + dataMaskPolicy.setResources(dataMaskResources) + RangerPolicy.RangerDataMaskPolicyItem dataMaskPolicyItem = new RangerPolicy.RangerDataMaskPolicyItem() + dataMaskPolicyItem.setUsers([user]) + dataMaskPolicyItem.setAccesses([new RangerPolicy.RangerPolicyItemAccess("select")]) + dataMaskPolicyItem.setDataMaskInfo( + new RangerPolicy.RangerPolicyItemDataMaskInfo("MASK_NULL", "", "")) + dataMaskPolicy.setDataMaskPolicyItems([dataMaskPolicyItem]) + rangerClient.createPolicy(dataMaskPolicy) + + // Recreate the catalog with Ranger-Hive enabled after policies are ready. + sql """CREATE CATALOG `${catalog}` PROPERTIES ( + 'type' = 'hms', + 'hive.metastore.uris' = 'thrift://${externalEnvIp}:${hmsPort}', + 'access_controller.properties.ranger.service.name' = '${rangerHiveServiceName}', + 'access_controller.class' = + 'org.apache.doris.catalog.authorizer.ranger.hive.RangerHiveAccessControllerFactory' + )""" + waitPolicyEffect() + + def tokens = context.config.jdbcUrl.split('/') + def defaultJdbcUrl = tokens[0] + "//" + tokens[2] + "/?" + connect(user, password, defaultJdbcUrl) { + String query = "SELECT id, secret FROM `${catalog}`.`${database}`.`${table}` ORDER BY id" + + // The first query initializes Ranger's optimized policy evaluator. The second query verifies + // that lowercase Hive access types still match after that evaluator has been warmed up. + order_qt_lowercase_access_type_first(query) + order_qt_lowercase_access_type_warmed_up(query) + } + } finally { + try_sql "DROP CATALOG IF EXISTS `${catalog}`" + policyNames.each { deletePolicyIfExists(it) } + try_sql "DROP USER IF EXISTS '${user}'" + if (createdHiveService) { + try { + rangerClient.deleteService(rangerHiveServiceName) + } catch (Exception e) { + logger.warn("Failed to delete Ranger-Hive service ${rangerHiveServiceName}: ${e.getMessage()}") + } + } + + try { + sql """DROP CATALOG IF EXISTS `${cleanupCatalog}`""" + sql """CREATE CATALOG `${cleanupCatalog}` PROPERTIES ( + 'type' = 'hms', + 'hive.metastore.uris' = 'thrift://${externalEnvIp}:${hmsPort}' + )""" + try_sql """DROP TABLE IF EXISTS `${cleanupCatalog}`.`${database}`.`${table}`""" + try_sql """DROP DATABASE IF EXISTS `${cleanupCatalog}`.`${database}`""" + } finally { + try_sql "DROP CATALOG IF EXISTS `${cleanupCatalog}`" + } + } +} From ed38b588aa0d4c0a525f4b02616500ac7834076a Mon Sep 17 00:00:00 2001 From: guoqiang Date: Fri, 28 Aug 2026 14:26:44 +0800 Subject: [PATCH 2/2] [test](regression) Preserve Ranger Hive regression state ### What problem does this PR solve? Issue Number: None Related PR: #67206 Problem Summary: The Ranger Hive lowercase regression performed throwing HMS catalog operations from its outer teardown. A teardown failure could replace the original regression failure, and successful teardown removed the Hive table and database needed for post-failure debugging. Remove the redundant post-test HMS cleanup and rely on the existing pre-test cleanup while retaining best-effort cleanup for transient Doris and Ranger resources. ### Release note None ### Check List (For Author) - Test: Regression test - test_ranger_hive_lowercase_access_type - Behavior changed: No - Does this need documentation: No --- ...test_ranger_hive_lowercase_access_type.groovy | 16 +--------------- 1 file changed, 1 insertion(+), 15 deletions(-) diff --git a/regression-test/suites/ranger_p2/test_ranger_hive_lowercase_access_type.groovy b/regression-test/suites/ranger_p2/test_ranger_hive_lowercase_access_type.groovy index 5639882edf2b28..876bd0b70193bc 100644 --- a/regression-test/suites/ranger_p2/test_ranger_hive_lowercase_access_type.groovy +++ b/regression-test/suites/ranger_p2/test_ranger_hive_lowercase_access_type.groovy @@ -39,7 +39,6 @@ suite("test_ranger_hive_lowercase_access_type", "p2,ranger,external") { String rangerHiveServiceName = "${rangerServiceName}_hive" String catalog = "ranger_hive_lowercase_catalog" - String cleanupCatalog = "ranger_hive_lowercase_cleanup_catalog" String database = "ranger_hive_lowercase_db" String table = "ranger_hive_lowercase_tbl" String user = "ranger_hive_lowercase_user" @@ -94,9 +93,8 @@ suite("test_ranger_hive_lowercase_access_type", "p2,ranger,external") { createdHiveService = true } - // Prepare the Hive data through a catalog without Ranger, so setup and cleanup do not need policies. + // Prepare the Hive data through a catalog without Ranger, so setup does not need policies. sql """DROP CATALOG IF EXISTS `${catalog}`""" - sql """DROP CATALOG IF EXISTS `${cleanupCatalog}`""" sql """CREATE CATALOG `${catalog}` PROPERTIES ( 'type' = 'hms', 'hive.metastore.uris' = 'thrift://${externalEnvIp}:${hmsPort}' @@ -186,17 +184,5 @@ suite("test_ranger_hive_lowercase_access_type", "p2,ranger,external") { logger.warn("Failed to delete Ranger-Hive service ${rangerHiveServiceName}: ${e.getMessage()}") } } - - try { - sql """DROP CATALOG IF EXISTS `${cleanupCatalog}`""" - sql """CREATE CATALOG `${cleanupCatalog}` PROPERTIES ( - 'type' = 'hms', - 'hive.metastore.uris' = 'thrift://${externalEnvIp}:${hmsPort}' - )""" - try_sql """DROP TABLE IF EXISTS `${cleanupCatalog}`.`${database}`.`${table}`""" - try_sql """DROP DATABASE IF EXISTS `${cleanupCatalog}`.`${database}`""" - } finally { - try_sql "DROP CATALOG IF EXISTS `${cleanupCatalog}`" - } } }