From 5933a417c77bcd7bbb26ca0ffaa1af740c70439a Mon Sep 17 00:00:00 2001 From: "zhanghaobo@kanzhun.com" Date: Mon, 22 Apr 2024 15:31:41 +0800 Subject: [PATCH 1/5] HDFS-17484. Introduce redundancy.considerLoad.minLoad to avoiding excluding nodes when they are not busy actually. --- .../main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java | 3 +++ .../blockmanagement/BlockPlacementPolicyDefault.java | 5 ++++- .../hadoop-hdfs/src/main/resources/hdfs-default.xml | 8 ++++++++ 3 files changed, 15 insertions(+), 1 deletion(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java index f92a2ad56581b9..8cb90bd47e2fd2 100755 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java @@ -293,6 +293,9 @@ public class DFSConfigKeys extends CommonConfigurationKeys { public static final boolean DFS_NAMENODE_REDUNDANCY_CONSIDERLOADBYVOLUME_DEFAULT = false; + public static final String DFS_NAMENODE_REDUNDANCY_CONSIDERLOAD_MINLOAD_KEY = + "dfs.namenode.redundancy.considerLoad.minload"; + public static final int DFS_NAMENODE_REDUNDANCY_CONSIDERLOAD_MINLOAD_DEFAULT = 16; public static final String DFS_NAMENODE_REDUNDANCY_INTERVAL_SECONDS_KEY = HdfsClientConfigKeys.DeprecatedKeys.DFS_NAMENODE_REDUNDANCY_INTERVAL_SECONDS_KEY; public static final int DFS_NAMENODE_REDUNDANCY_INTERVAL_SECONDS_DEFAULT = 3; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java index 8020d7c45b37ad..31bf881adb586f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java @@ -105,6 +105,7 @@ private String getText() { private boolean considerLoadByStorageType; protected double considerLoadFactor; private boolean considerLoadByVolume = false; + protected int considerLoadMinLoad; private boolean preferLocalNode; private boolean dataNodePeerStatsEnabled; private volatile boolean excludeSlowNodesEnabled; @@ -140,6 +141,8 @@ public void initialize(Configuration conf, FSClusterStats stats, DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_CONSIDERLOADBYVOLUME_KEY, DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_CONSIDERLOADBYVOLUME_DEFAULT ); + this.considerLoadMinLoad = conf.getInt(DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_CONSIDERLOAD_MINLOAD_KEY, + DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_CONSIDERLOAD_MINLOAD_DEFAULT); this.stats = stats; this.clusterMap = clusterMap; this.host2datanodeMap = host2datanodeMap; @@ -1014,7 +1017,7 @@ boolean excludeNodeByLoad(DatanodeDescriptor node){ final double maxLoad = considerLoadFactor * inServiceXceiverCount; final int nodeLoad = node.getXceiverCount(); - if ((nodeLoad > maxLoad) && (maxLoad > 0)) { + if ((nodeLoad > considerLoadMinLoad) &&(nodeLoad > maxLoad) && (maxLoad > 0)) { logNodeIsNotChosen(node, NodeNotChosenReason.NODE_TOO_BUSY, "(load: " + nodeLoad + " > " + maxLoad + ")"); return true; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml index e6dc8c5ba1ac42..86bce71ba526e2 100755 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml @@ -334,6 +334,14 @@ + + dfs.namenode.redundancy.considerLoad.minLoad + 16 + The minimum load which a node's load must exceed + before being rejected for writes, only if considerLoad is true. + + + dfs.namenode.redundancy.considerLoadByVolume false From 40ea0b462e83570e267fe48776c3f000d2686646 Mon Sep 17 00:00:00 2001 From: "zhanghaobo@kanzhun.com" Date: Mon, 22 Apr 2024 15:39:20 +0800 Subject: [PATCH 2/5] fix unit tests failed. --- .../hdfs/server/blockmanagement/TestReplicationPolicy.java | 1 + 1 file changed, 1 insertion(+) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java index b99e060ee387ca..bd43cf09aa5cf9 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java @@ -1662,6 +1662,7 @@ public void testMaxLoad() { when(node.getXceiverCount()).thenReturn(1); final Configuration conf = new Configuration(); + conf.setInt(DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_CONSIDERLOAD_MINLOAD_KEY, 1); final Class replicatorClass = conf .getClass(DFSConfigKeys.DFS_BLOCK_REPLICATOR_CLASSNAME_KEY, DFSConfigKeys.DFS_BLOCK_REPLICATOR_CLASSNAME_DEFAULT, From b01d3bcf66844d7f7f66c04a496d8dcc39cddae6 Mon Sep 17 00:00:00 2001 From: "zhanghaobo@kanzhun.com" Date: Mon, 22 Apr 2024 15:51:42 +0800 Subject: [PATCH 3/5] add an unit test. --- .../TestReplicationPolicy.java | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java index bd43cf09aa5cf9..ab3254cb9ceda6 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java @@ -1709,6 +1709,32 @@ public void testMaxLoad() { assertFalse(bppd.excludeNodeByLoad(node)); } + @Test + public void testMinLoad() { + FSClusterStats statistics = mock(FSClusterStats.class); + DatanodeDescriptor node = mock(DatanodeDescriptor.class); + + when(statistics.getInServiceXceiverAverage()).thenReturn(5D); + when(node.getXceiverCount()).thenReturn(12); + + final Configuration conf = new Configuration(); + conf.setInt(DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_CONSIDERLOAD_MINLOAD_KEY, 16); + final Class replicatorClass = conf + .getClass(DFSConfigKeys.DFS_BLOCK_REPLICATOR_CLASSNAME_KEY, + DFSConfigKeys.DFS_BLOCK_REPLICATOR_CLASSNAME_DEFAULT, + BlockPlacementPolicy.class); + BlockPlacementPolicy bpp = ReflectionUtils. + newInstance(replicatorClass, conf); + assertTrue(bpp instanceof BlockPlacementPolicyDefault); + + BlockPlacementPolicyDefault bppd = (BlockPlacementPolicyDefault) bpp; + bppd.initialize(conf, statistics, null, null); + assertFalse(bppd.excludeNodeByLoad(node)); + + when(node.getXceiverCount()).thenReturn(17); + assertTrue(bppd.excludeNodeByLoad(node)); + } + @Test public void testChosenFailureForStorageType() { final LogVerificationAppender appender = new LogVerificationAppender(); From 9cceb198d62d185f49bfd65ce5641c39035f5188 Mon Sep 17 00:00:00 2001 From: "zhanghaobo@kanzhun.com" Date: Tue, 23 Apr 2024 10:39:40 +0800 Subject: [PATCH 4/5] fix blanks --- .../hdfs/server/blockmanagement/TestReplicationPolicy.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java index ab3254cb9ceda6..868cc0d8c4a348 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java @@ -1734,7 +1734,7 @@ public void testMinLoad() { when(node.getXceiverCount()).thenReturn(17); assertTrue(bppd.excludeNodeByLoad(node)); } - + @Test public void testChosenFailureForStorageType() { final LogVerificationAppender appender = new LogVerificationAppender(); From 0b75e7bd36f06728bcf31f08686c476d580270eb Mon Sep 17 00:00:00 2001 From: "zhanghaobo@kanzhun.com" Date: Tue, 23 Apr 2024 17:07:13 +0800 Subject: [PATCH 5/5] fix failed uts. --- .../server/blockmanagement/BlockPlacementPolicyDefault.java | 3 ++- .../hadoop-hdfs/src/main/resources/hdfs-default.xml | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java index 31bf881adb586f..348aac33bfcaf2 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java @@ -141,7 +141,8 @@ public void initialize(Configuration conf, FSClusterStats stats, DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_CONSIDERLOADBYVOLUME_KEY, DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_CONSIDERLOADBYVOLUME_DEFAULT ); - this.considerLoadMinLoad = conf.getInt(DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_CONSIDERLOAD_MINLOAD_KEY, + this.considerLoadMinLoad = conf.getInt( + DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_CONSIDERLOAD_MINLOAD_KEY, DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_CONSIDERLOAD_MINLOAD_DEFAULT); this.stats = stats; this.clusterMap = clusterMap; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml index 86bce71ba526e2..13f2eb9f371cf1 100755 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml @@ -335,7 +335,7 @@ - dfs.namenode.redundancy.considerLoad.minLoad + dfs.namenode.redundancy.considerLoad.minload 16 The minimum load which a node's load must exceed before being rejected for writes, only if considerLoad is true.