[fix](data security) Mask ai.api_key in CREATE/ALTER RESOURCE logs

fe/fe-core/src/main/java/org/apache/doris/catalog/ResourceMgr.java

@@ -154,7 +154,8 @@ public void alterResource(String resourceName, Map<String, String> properties) t
 
  // log alter
  Env.getCurrentEnv().getEditLog().logAlterResource(resource);
- LOG.info("Alter resource success. Resource: {}", resource);
+ // Only log non-sensitive identifiers here because resource objects may retain credential properties.
+ LOG.info("Alter resource success. Resource: {}, type: {}", resource.getName(), resource.getType());
  }
 
  public void replayAlterResource(Resource resource) {

fe/fe-core/src/main/java/org/apache/doris/common/util/DatasourcePrintableMap.java

@@ -55,6 +55,7 @@ public class DatasourcePrintableMap<K, V> extends BasicPrintableMap<K, V> {
  SENSITIVE_KEY.add("bos_secret_accesskey");
  SENSITIVE_KEY.add("jdbc.password");
  SENSITIVE_KEY.add("elasticsearch.password");
+ SENSITIVE_KEY.add("ai.api_key");
  SENSITIVE_KEY.addAll(Arrays.asList(
  MCProperties.SECRET_KEY));
  SENSITIVE_KEY.addAll(ConnectorPropertiesUtils.getSensitiveKeys(S3Properties.class));

fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilderForEncryption.java

@@ -24,7 +24,6 @@
 import org.apache.doris.nereids.DorisParser;
 import org.apache.doris.nereids.DorisParser.InsertTableContext;
 import org.apache.doris.nereids.DorisParser.JobFromToClauseContext;
-import org.apache.doris.nereids.DorisParser.SupportedDmlStatementContext;
 import org.apache.doris.nereids.trees.plans.commands.info.SetVarOp;
 import org.apache.doris.nereids.trees.plans.logical.LogicalPlan;
 
@@ -203,6 +202,17 @@ public LogicalPlan visitAlterAuthenticationIntegrationProperties(
  return super.visitAlterAuthenticationIntegrationProperties(ctx);
  }
 
+ @Override
+ public LogicalPlan visitAlterResource(DorisParser.AlterResourceContext ctx) {
+ if (ctx.propertyClause() != null) {
+ DorisParser.PropertyClauseContext propertyClauseContext = ctx.propertyClause();
+ encryptProperty(visitPropertyClause(propertyClauseContext),
+ propertyClauseContext.fileProperties.start.getStartIndex(),
+ propertyClauseContext.fileProperties.stop.getStopIndex());
+ }
+ return super.visitAlterResource(ctx);
+ }
+
  // select from tvf
  @Override
  public LogicalPlan visitTableValuedFunction(DorisParser.TableValuedFunctionContext ctx) {
@@ -217,9 +227,8 @@ public LogicalPlan visitTableValuedFunction(DorisParser.TableValuedFunctionConte
  // create job select tvf
  @Override
  public LogicalPlan visitCreateScheduledJob(DorisParser.CreateScheduledJobContext ctx) {
- if (ctx.supportedDmlStatement() != null) {
- SupportedDmlStatementContext supportedDmlStatementContext = ctx.supportedDmlStatement();
- visitInsertTable((InsertTableContext) supportedDmlStatementContext);
+ if (ctx.supportedDmlStatement() instanceof InsertTableContext) {
+ visitInsertTable((InsertTableContext) ctx.supportedDmlStatement());
  } else if (ctx.jobFromToClause() != null) {
  JobFromToClauseContext jobFromToClauseContext = ctx.jobFromToClause();
  encryptProperty(visitPropertyItemList(jobFromToClauseContext.sourceProperties),
@@ -233,9 +242,8 @@ public LogicalPlan visitCreateScheduledJob(DorisParser.CreateScheduledJobContext
  // alter job select tvf
  @Override
  public LogicalPlan visitAlterJob(DorisParser.AlterJobContext ctx) {
- SupportedDmlStatementContext supportedDmlStatementContext = ctx.supportedDmlStatement();
- if (ctx.supportedDmlStatement() != null) {
- visitInsertTable((InsertTableContext) supportedDmlStatementContext);
+ if (ctx.supportedDmlStatement() instanceof InsertTableContext) {
+ visitInsertTable((InsertTableContext) ctx.supportedDmlStatement());
  } else if (ctx.jobFromToClause() != null) {
  JobFromToClauseContext jobFromToClauseContext = ctx.jobFromToClause();
  encryptProperty(visitPropertyItemList(jobFromToClauseContext.sourceProperties),
@@ -246,6 +254,17 @@ public LogicalPlan visitAlterJob(DorisParser.AlterJobContext ctx) {
  return super.visitAlterJob(ctx);
  }
 
+ @Override
+ public LogicalPlan visitCreateResource(DorisParser.CreateResourceContext ctx) {
+ if (ctx.properties != null) {
+ DorisParser.PropertyClauseContext propertyClauseContext = ctx.properties;
+ encryptProperty(visitPropertyClause(propertyClauseContext),
+ propertyClauseContext.fileProperties.start.getStartIndex(),
+ propertyClauseContext.fileProperties.stop.getStopIndex());
+ }
+ return super.visitCreateResource(ctx);
+ }
+
  private void encryptProperty(Map<String, String> properties, int start, int stop) {
  if (MapUtils.isNotEmpty(properties)) {
  DatasourcePrintableMap<String, String> printableMap = new DatasourcePrintableMap<>(properties, "=",

fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java

@@ -100,6 +100,7 @@
 import org.apache.doris.nereids.trees.plans.commands.EmptyCommand;
 import org.apache.doris.nereids.trees.plans.commands.Forward;
 import org.apache.doris.nereids.trees.plans.commands.LoadCommand;
+import org.apache.doris.nereids.trees.plans.commands.NeedAuditEncryption;
 import org.apache.doris.nereids.trees.plans.commands.PrepareCommand;
 import org.apache.doris.nereids.trees.plans.commands.Redirect;
 import org.apache.doris.nereids.trees.plans.commands.SupportProfile;
@@ -182,6 +183,7 @@ public class StmtExecutor {
  private static final Logger LOG = LogManager.getLogger(StmtExecutor.class);
 
  private static final AtomicLong STMT_ID_GENERATOR = new AtomicLong(0);
+ private static final String MASKED_STMT_FALLBACK = "/* masked statement unavailable */";
  public static final int MAX_DATA_TO_SEND_FOR_TXN = 100;
  private static Set<String> blockSqlAstNames = Sets.newHashSet();
 
@@ -588,7 +590,7 @@ public void execute() throws Exception {
  TUniqueId queryId = UniqueIdUtils.fastUniqueId();
  if (Config.enable_print_request_before_execution) {
  LOG.info("begin to execute query {} {}",
- DebugUtil.printId(queryId), originStmt == null ? "null" : originStmt.originStmt);
+ DebugUtil.printId(queryId), getStmtForLoggingBeforeParse());
  }
  queryRetry(queryId);
  }
@@ -756,7 +758,7 @@ public void checkBlockRulesByScan(Planner planner) throws AnalysisException {
 
  private void executeByNereids(TUniqueId queryId) throws Exception {
  if (LOG.isDebugEnabled()) {
- LOG.debug("Nereids start to execute query:\n {}", originStmt.originStmt);
+ LOG.debug("Nereids start to execute query:\n {}", getStmtForLoggingBeforeParse());
  }
  context.setQueryId(queryId);
  context.setStartTime();
@@ -837,29 +839,32 @@ private void executeByNereids(TUniqueId queryId) throws Exception {
  ((Command) logicalPlan).run(context, this);
  } catch (QueryStateException e) {
  if (LOG.isDebugEnabled()) {
- LOG.debug("Command({}) process failed.", originStmt.originStmt, e);
+ LOG.debug("Command({}) process failed.", getStmtForLogging(originStmt.originStmt), e);
  }
  context.setState(e.getQueryState());
- throw new NereidsException("Command(" + originStmt.originStmt + ") process failed",
+ throw new NereidsException("Command(" + getStmtForLogging(originStmt.originStmt)
+ + ") process failed",
  new AnalysisException(e.getMessage(), e));
  } catch (UserException e) {
  // Return message to info client what happened.
  if (LOG.isDebugEnabled()) {
- LOG.debug("Command({}) process failed.", originStmt.originStmt, e);
+ LOG.debug("Command({}) process failed.", getStmtForLogging(originStmt.originStmt), e);
  }
  if (Config.isCloudMode() && SystemInfoService.needRetryWithReplan(e.getDetailMessage())) {
  // For errors in SystemInfoService.NEED_REPLAN_ERRORS,
  // throw exception directly to trigger a replan retry outside(in StmtExecutor.queryRetry())
  throw e;
  }
  context.getState().setError(e.getMysqlErrorCode(), e.getMessage());
- throw new NereidsException("Command (" + originStmt.originStmt + ") process failed",
+ throw new NereidsException("Command (" + getStmtForLogging(originStmt.originStmt)
+ + ") process failed",
  new AnalysisException(e.getMessage(), e));
  } catch (Exception | Error e) {
  // Maybe our bug
- LOG.info("Command({}) process failed.", originStmt.originStmt, e);
+ LOG.info("Command({}) process failed.", getStmtForLogging(originStmt.originStmt), e);
  context.getState().setError(ErrorCode.ERR_UNKNOWN_ERROR, e.getMessage());
- throw new NereidsException("Command (" + originStmt.originStmt + ") process failed.",
+ throw new NereidsException("Command (" + getStmtForLogging(originStmt.originStmt)
+ + ") process failed.",
  new AnalysisException(e.getMessage() == null ? e.toString() : e.getMessage(), e));
  }
  } else {
@@ -899,7 +904,7 @@ private void executeByNereids(TUniqueId queryId) throws Exception {
  planner.plan(parsedStmt, context.getSessionVariable().toThrift());
  checkBlockRulesByScan(planner);
  } catch (Exception e) {
- LOG.warn("Nereids plan query failed:\n{}", originStmt.originStmt, e);
+ LOG.warn("Nereids plan query failed:\n{}", getStmtForLogging(originStmt.originStmt), e);
  throw new NereidsException(new AnalysisException(e.getMessage(), e));
  }
  profile.getSummaryProfile().setQueryPlanFinishTime(TimeUtils.getStartTimeMs());
@@ -2390,6 +2395,55 @@ public String getOriginStmtInString() {
  return "";
  }
 
+ private String getStmtForLogging(String stmt) {
+ if (stmt == null) {
+ return stmt;
+ }
+ if (!(parsedStmt instanceof LogicalPlanAdapter)) {
+ return getStmtForLoggingBeforeParse(stmt);
+ }
+ // Internal export outfile tasks use an empty origin SQL, so audit masking must skip reparsing here.
+ if (stmt.isEmpty()) {
+ return stmt;
+ }
+ LogicalPlan logicalPlan = ((LogicalPlanAdapter) parsedStmt).getLogicalPlan();
+ if (!(logicalPlan instanceof NeedAuditEncryption)) {
+ return stmt;
+ }
+ try {
+ return ((NeedAuditEncryption) logicalPlan).geneEncryptionSQL(stmt);
+ } catch (Exception e) {
+ // Logging must not leak plaintext or change command behavior when masking fails.
+ LOG.warn("failed to mask statement for FE logging", e);
+ return MASKED_STMT_FALLBACK;
+ }
+ }
+
+ private String getStmtForLoggingBeforeParse() {
+ return getStmtForLoggingBeforeParse(originStmt == null ? null : originStmt.originStmt);
+ }
+
+ private String getStmtForLoggingBeforeParse(String stmt) {
+ if (stmt == null) {
+ return null;
+ }
+ // Empty SQL cannot produce a valid parse tree for audit masking, so keep the original text.
+ if (stmt.isEmpty()) {
+ return stmt;
+ }
+ try {
+ LogicalPlan logicalPlan = new NereidsParser().parseSingle(stmt);
+ if (!(logicalPlan instanceof NeedAuditEncryption)) {
+ return stmt;
+ }
+ return ((NeedAuditEncryption) logicalPlan).geneEncryptionSQL(stmt);
+ } catch (Exception e) {
+ // Logging must fail closed before parsing so secrets never fall back to plaintext.
+ LOG.warn("failed to prepare masked statement for FE logging", e);
+ return MASKED_STMT_FALLBACK;
+ }
+ }
+
  public List<ByteBuffer> getProxyQueryResultBufList() {
  return ((ProxyMysqlChannel) context.getMysqlChannel()).getProxyResultBufferList();
  }

fe/fe-core/src/test/java/org/apache/doris/nereids/parser/RepositoryAuditEncryptionTest.java

@@ -60,4 +60,39 @@ public void testAlterRepositoryMasksSecretKey() {
  Assertions.assertFalse(masked.contains("SUPERSECRET"), "secret_key must be masked: " + masked);
  Assertions.assertTrue(masked.contains("*XXX"), "expected mask token: " + masked);
  }
+
+ @Test
+ public void testCreateResourceMasksAiApiKey() {
+ String sql = "CREATE EXTERNAL RESOURCE \"ai_resource\" PROPERTIES ("
+ + "\"type\" = \"ai\", "
+ + "\"ai.api_key\" = \"sk-test\", "
+ + "\"ai.endpoint\" = \"https://api.test\")";
+ String masked = encrypt(sql);
+ Assertions.assertFalse(masked.contains("sk-test"), masked);
+ Assertions.assertTrue(masked.contains("*XXX"), masked);
+ Assertions.assertTrue(masked.contains("https://api.test"), masked);
+ }
+
+ @Test
+ public void testAlterResourceMasksAiApiKey() {
+ String sql = "ALTER RESOURCE \"ai_resource\" PROPERTIES ("
+ + "\"ai.api_key\" = \"sk-test\", "
+ + "\"ai.endpoint\" = \"https://api.test\")";
+ String masked = encrypt(sql);
+ Assertions.assertFalse(masked.contains("sk-test"), masked);
+ Assertions.assertTrue(masked.contains("*XXX"), masked);
+ Assertions.assertTrue(masked.contains("https://api.test"), masked);
+ }
+
+ @Test
+ public void testCreateJobWithUpdateDoesNotThrowClassCastException() {
+ // CREATE JOB ... DO <update> parses the DML as UpdateContext, not InsertTableContext.
+ // geneEncryptionSQL must not fail with ClassCastException on such statements.
+ String sql = "CREATE JOB job1 ON SCHEDULE AT CURRENT_TIMESTAMP DO UPDATE t SET type = 2 WHERE type = 1";
+ LogicalPlan plan = new NereidsParser().parseSingle(sql);
+ Assertions.assertTrue(plan instanceof NeedAuditEncryption,
+ "command should be NeedAuditEncryption: " + plan.getClass().getName());
+ NeedAuditEncryption cmd = (NeedAuditEncryption) plan;
+ Assertions.assertDoesNotThrow(() -> cmd.geneEncryptionSQL(sql));
+ }
 }