Skip to content

HDFS-17484. Introduce redundancy.considerLoad.minLoad to avoiding excluding nodes when they are not busy actually. #6758

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
hfutatzhanghb wants to merge 5 commits into from
Closed

HDFS-17484. Introduce redundancy.considerLoad.minLoad to avoiding excluding nodes when they are not busy actually. #6758

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
5933a41
HDFS-17484. Introduce redundancy.considerLoad.minLoad to avoiding exc…
hfutatzhanghb Apr 22, 2024
40ea0b4
fix unit tests failed.
hfutatzhanghb Apr 22, 2024
b01d3bc
add an unit test.
hfutatzhanghb Apr 22, 2024
9cceb19
fix blanks
hfutatzhanghb Apr 23, 2024
0b75e7b
fix failed uts.
hfutatzhanghb Apr 23, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -293,6 +293,9 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
public static final boolean
DFS_NAMENODE_REDUNDANCY_CONSIDERLOADBYVOLUME_DEFAULT
= false;
public static final String DFS_NAMENODE_REDUNDANCY_CONSIDERLOAD_MINLOAD_KEY =
"dfs.namenode.redundancy.considerLoad.minload";
public static final int DFS_NAMENODE_REDUNDANCY_CONSIDERLOAD_MINLOAD_DEFAULT = 16;
public static final String DFS_NAMENODE_REDUNDANCY_INTERVAL_SECONDS_KEY =
HdfsClientConfigKeys.DeprecatedKeys.DFS_NAMENODE_REDUNDANCY_INTERVAL_SECONDS_KEY;
public static final int DFS_NAMENODE_REDUNDANCY_INTERVAL_SECONDS_DEFAULT = 3;
Expand Down
6 changes: 5 additions & 1 deletion .../main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -105,6 +105,7 @@ private String getText() {
private boolean considerLoadByStorageType;
protected double considerLoadFactor;
private boolean considerLoadByVolume = false;
protected int considerLoadMinLoad;
private boolean preferLocalNode;
private boolean dataNodePeerStatsEnabled;
private volatile boolean excludeSlowNodesEnabled;
Expand Down Expand Up @@ -140,6 +141,9 @@ public void initialize(Configuration conf, FSClusterStats stats,
DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_CONSIDERLOADBYVOLUME_KEY,
DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_CONSIDERLOADBYVOLUME_DEFAULT
);
this.considerLoadMinLoad = conf.getInt(
DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_CONSIDERLOAD_MINLOAD_KEY,
DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_CONSIDERLOAD_MINLOAD_DEFAULT);
this.stats = stats;
this.clusterMap = clusterMap;
this.host2datanodeMap = host2datanodeMap;
Expand Down Expand Up @@ -1014,7 +1018,7 @@ boolean excludeNodeByLoad(DatanodeDescriptor node){
final double maxLoad = considerLoadFactor * inServiceXceiverCount;

final int nodeLoad = node.getXceiverCount();
if ((nodeLoad > maxLoad) && (maxLoad > 0)) {
if ((nodeLoad > considerLoadMinLoad) &&(nodeLoad > maxLoad) && (maxLoad > 0)) {
logNodeIsNotChosen(node, NodeNotChosenReason.NODE_TOO_BUSY,
"(load: " + nodeLoad + " > " + maxLoad + ")");
return true;
Expand Down
8 changes: 8 additions & 0 deletions hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -334,6 +334,14 @@
</description>
</property>

<property>
<name>dfs.namenode.redundancy.considerLoad.minload</name>
<value>16</value>
<description>The minimum load which a node's load must exceed
before being rejected for writes, only if considerLoad is true.
</description>
</property>

<property>
<name>dfs.namenode.redundancy.considerLoadByVolume</name>
<value>false</value>
Expand Down
27 changes: 27 additions & 0 deletions ...fs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1662,6 +1662,7 @@ public void testMaxLoad() {
when(node.getXceiverCount()).thenReturn(1);

final Configuration conf = new Configuration();
conf.setInt(DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_CONSIDERLOAD_MINLOAD_KEY, 1);
final Class<? extends BlockPlacementPolicy> replicatorClass = conf
.getClass(DFSConfigKeys.DFS_BLOCK_REPLICATOR_CLASSNAME_KEY,
DFSConfigKeys.DFS_BLOCK_REPLICATOR_CLASSNAME_DEFAULT,
Expand Down Expand Up @@ -1708,6 +1709,32 @@ public void testMaxLoad() {
assertFalse(bppd.excludeNodeByLoad(node));
}

@Test
public void testMinLoad() {
FSClusterStats statistics = mock(FSClusterStats.class);
DatanodeDescriptor node = mock(DatanodeDescriptor.class);

when(statistics.getInServiceXceiverAverage()).thenReturn(5D);
when(node.getXceiverCount()).thenReturn(12);

final Configuration conf = new Configuration();
conf.setInt(DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_CONSIDERLOAD_MINLOAD_KEY, 16);
final Class<? extends BlockPlacementPolicy> replicatorClass = conf
.getClass(DFSConfigKeys.DFS_BLOCK_REPLICATOR_CLASSNAME_KEY,
DFSConfigKeys.DFS_BLOCK_REPLICATOR_CLASSNAME_DEFAULT,
BlockPlacementPolicy.class);
BlockPlacementPolicy bpp = ReflectionUtils.
newInstance(replicatorClass, conf);
assertTrue(bpp instanceof BlockPlacementPolicyDefault);

BlockPlacementPolicyDefault bppd = (BlockPlacementPolicyDefault) bpp;
bppd.initialize(conf, statistics, null, null);
assertFalse(bppd.excludeNodeByLoad(node));

when(node.getXceiverCount()).thenReturn(17);
assertTrue(bppd.excludeNodeByLoad(node));
}

@Test
public void testChosenFailureForStorageType() {
final LogVerificationAppender appender = new LogVerificationAppender();
Expand Down