From 1ee7d6ef6d167ba55233834488237e289038a181 Mon Sep 17 00:00:00 2001 From: zhanghaobo Date: Wed, 21 May 2025 21:40:33 +0800 Subject: [PATCH 1/3] HDFS-17788. [ARR] getFileInfo not handle exception rightly which may cause FileNotFoundException in DistributedFileSystem. --- .../async/RouterAsyncClientProtocol.java | 2 + .../TestRouterAsyncClientRejectOverload.java | 81 +++++++++++++++++++ 2 files changed, 83 insertions(+) create mode 100644 hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/async/TestRouterAsyncClientRejectOverload.java diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/async/RouterAsyncClientProtocol.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/async/RouterAsyncClientProtocol.java index 15c6cbbfd271d4..d3903cbfef75dd 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/async/RouterAsyncClientProtocol.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/async/RouterAsyncClientProtocol.java @@ -579,6 +579,8 @@ public HdfsFileStatus getFileInfo(String src) throws IOException { if (e instanceof NoLocationException || e instanceof RouterResolveException) { noLocationException[0] = e; + } else { + throw e; } return null; }, IOException.class); diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/async/TestRouterAsyncClientRejectOverload.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/async/TestRouterAsyncClientRejectOverload.java new file mode 100644 index 00000000000000..dd639817e4aebd --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/async/TestRouterAsyncClientRejectOverload.java @@ -0,0 +1,81 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs.server.federation.router.async; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hdfs.DFSClient; +import org.apache.hadoop.hdfs.protocol.DirectoryListing; +import org.apache.hadoop.hdfs.protocol.HdfsFileStatus; +import org.apache.hadoop.hdfs.server.federation.RouterConfigBuilder; +import org.apache.hadoop.hdfs.server.federation.StateStoreDFSCluster; +import org.apache.hadoop.hdfs.server.federation.router.RBFConfigKeys; +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.net.URI; + +import static org.apache.hadoop.hdfs.server.federation.FederationTestUtils.transitionClusterNSToActive; +import static org.apache.hadoop.hdfs.server.federation.FederationTestUtils.transitionClusterNSToStandby; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +public class TestRouterAsyncClientRejectOverload { + + private StateStoreDFSCluster cluster; + + private void setupCluster(boolean ha) + throws Exception { + // Build and start a federated cluster. + cluster = new StateStoreDFSCluster(ha, 2); + Configuration routerConf = new RouterConfigBuilder() + .stateStore() + .metrics() + .admin() + .rpc() + .heartbeat() + .build(); + + routerConf.setBoolean(RBFConfigKeys.DFS_ROUTER_ASYNC_RPC_ENABLE_KEY, true); + + cluster.addRouterOverrides(routerConf); + cluster.startCluster(); + cluster.startRouters(); + cluster.waitClusterUp(); + } + + @Test + public void testGetFileInfoWhenNsFailover() throws Exception { + setupCluster(true); + Configuration conf = cluster.getRouterClientConf(); + conf.setInt("dfs.client.retry.max.attempts", 2); + DFSClient routerClient = new DFSClient(new URI("hdfs://fed"), conf); + transitionClusterNSToActive(cluster, 0); + + String basePath = "/ARR/testGetFileInfo"; + routerClient.mkdirs(basePath); + DirectoryListing directoryListing = routerClient.listPaths("/ARR", new byte[0]); + assertEquals(1, directoryListing.getPartialListing().length); + + transitionClusterNSToStandby(cluster); + + assertThrows(IOException.class, () -> { + HdfsFileStatus fileInfo = routerClient.getFileInfo(basePath + 1); + }); + } +} + From dcb7509b8418635416a16219e58a6d714b21ba8c Mon Sep 17 00:00:00 2001 From: zhanghaobo Date: Thu, 22 May 2025 14:39:20 +0800 Subject: [PATCH 2/3] Rename file name --- ...verload.java => TestRouterAsyncRpcWhenNamenodeFailover.java} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/async/{TestRouterAsyncClientRejectOverload.java => TestRouterAsyncRpcWhenNamenodeFailover.java} (98%) diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/async/TestRouterAsyncClientRejectOverload.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/async/TestRouterAsyncRpcWhenNamenodeFailover.java similarity index 98% rename from hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/async/TestRouterAsyncClientRejectOverload.java rename to hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/async/TestRouterAsyncRpcWhenNamenodeFailover.java index dd639817e4aebd..cde63c0dbc7b00 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/async/TestRouterAsyncClientRejectOverload.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/async/TestRouterAsyncRpcWhenNamenodeFailover.java @@ -34,7 +34,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; -public class TestRouterAsyncClientRejectOverload { +public class TestRouterAsyncRpcWhenNamenodeFailover { private StateStoreDFSCluster cluster; From e9a14fdd312ae09e11ece967e7e56641f256962b Mon Sep 17 00:00:00 2001 From: zhanghaobo Date: Thu, 22 May 2025 15:06:02 +0800 Subject: [PATCH 3/3] fix checkstyle. --- .../router/async/TestRouterAsyncRpcWhenNamenodeFailover.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/async/TestRouterAsyncRpcWhenNamenodeFailover.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/async/TestRouterAsyncRpcWhenNamenodeFailover.java index cde63c0dbc7b00..64714c65d827f0 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/async/TestRouterAsyncRpcWhenNamenodeFailover.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/async/TestRouterAsyncRpcWhenNamenodeFailover.java @@ -67,7 +67,7 @@ public void testGetFileInfoWhenNsFailover() throws Exception { transitionClusterNSToActive(cluster, 0); String basePath = "/ARR/testGetFileInfo"; - routerClient.mkdirs(basePath); + routerClient.mkdirs(basePath); DirectoryListing directoryListing = routerClient.listPaths("/ARR", new byte[0]); assertEquals(1, directoryListing.getPartialListing().length);