From f7205907d306c4f91ba0088984400f17ec2e8c4f Mon Sep 17 00:00:00 2001 From: Susheel Gupta Date: Mon, 20 Jul 2026 17:51:51 +0530 Subject: [PATCH] YARN-11682: Legacy auto created queue in absolute mode has zero capacity after creation during app recovery --- .../capacity/ManagedParentQueue.java | 20 +++++++ ...uaranteedOrZeroCapacityOverTimePolicy.java | 25 +++++++- .../TestAbsoluteResourceWithAutoQueue.java | 60 +++++++++++++++++++ 3 files changed, 103 insertions(+), 2 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ManagedParentQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ManagedParentQueue.java index fa749f077964eb..8170e4bc3fe50b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ManagedParentQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ManagedParentQueue.java @@ -262,6 +262,26 @@ private void updateQueueCapacities(QueueCapacities queueCapacities) { } } +/** + * Recomputes leaf queue template capacities in ABSOLUTE_RESOURCE mode. + * This handles the case where a queue is recreated during RM recovery + * before any NodeManager has registered, causing the initial capacity + * to be computed as 0. The periodic queue management policy recalculates + * the template once cluster resources become available. No operation for other + * queue modes. + */ + public void updateTemplateCapacitiesForAbsoluteResource() { + writeLock.lock(); + try { + if (this.capacityConfigType.equals( + CapacityConfigType.ABSOLUTE_RESOURCE)) { + updateQueueCapacities(getLeafQueueTemplate().getQueueCapacities()); + } + } finally { + writeLock.unlock(); + } + } + protected void validate(final CSQueue newlyParsedQueue) throws IOException { // Sanity check if (!(newlyParsedQueue instanceof ManagedParentQueue) || !newlyParsedQueue diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/queuemanagement/GuaranteedOrZeroCapacityOverTimePolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/queuemanagement/GuaranteedOrZeroCapacityOverTimePolicy.java index c4579b24dafdff..03ae36a21ef058 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/queuemanagement/GuaranteedOrZeroCapacityOverTimePolicy.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/queuemanagement/GuaranteedOrZeroCapacityOverTimePolicy.java @@ -51,6 +51,8 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.locks.ReentrantReadWriteLock; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler + .capacity.AbstractCSQueue.CapacityConfigType.ABSOLUTE_RESOURCE; import static org.apache.hadoop.yarn.server.resourcemanager.scheduler .capacity.CSQueueUtils.EPSILON; @@ -310,6 +312,15 @@ private void initializeLeafQueueTemplate(ManagedParentQueue parentQueue) public List computeQueueManagementChanges() throws SchedulerDynamicEditException { + // Recompute the leaf queue template capacities from the current cluster + // resource for ABSOLUTE_RESOURCE mode. The template fraction is otherwise + // computed only at reinitialize; if a leaf queue was auto-created + // while the cluster resource was zero during RM recovery before any + // NodeManager registered the fraction stays 0 and the queue never regains + // capacity. This refresh lets the template pick up the real capacity once + // NodeManagers register. No operation unless it is ABSOLUTE_RESOURCE. + managedParentQueue.updateTemplateCapacitiesForAbsoluteResource(); + // Update template absolute capacities as the capacities could have changed // in weight mode updateTemplateAbsoluteCapacities(managedParentQueue.getQueueCapacities(), @@ -716,8 +727,18 @@ public AutoCreatedLeafQueueConfig getInitialLeafQueueConfiguration( getAbsoluteCapacity(nodeLabel) - parentQueueState. getAbsoluteActivatedChildQueueCapacity(nodeLabel) + EPSILON; - if (availableCapacity >= leafQueueTemplateCapacities - .getAbsoluteCapacity(nodeLabel)) { + // In ABSOLUTE_RESOURCE mode, don't activate a leaf queue while the cluster + // resource is 0 during RM recovery before NodeManagers register. + // Otherwise, it gets initialized with zero capacity and never updates. + // Create it deactivated instead, and let the monitor activate it once + // cluster resources are available. Percentage and weight modes are unaffected. + boolean deferActivationUntilClusterResource = + managedParentQueue.getCapacityConfigType() == ABSOLUTE_RESOURCE + && managedParentQueue.getQueueContext().getClusterResource() + .getMemorySize() <= 0; + + if (!deferActivationUntilClusterResource && availableCapacity + >= leafQueueTemplateCapacities.getAbsoluteCapacity(nodeLabel)) { updateCapacityFromTemplate(capacities, nodeLabel); activate(leafQueue, nodeLabel); } else{ diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAbsoluteResourceWithAutoQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAbsoluteResourceWithAutoQueue.java index ca3e52fec1512c..28b59dbc544340 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAbsoluteResourceWithAutoQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAbsoluteResourceWithAutoQueue.java @@ -21,6 +21,7 @@ import static org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager.NO_LABEL; import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueueUtils.EPSILON; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -338,4 +339,63 @@ public void testApplicationRunningWithDRF() throws Exception { ManagedParentQueue parentQueue = (ManagedParentQueue) cs.getQueue(QUEUED); assertEquals(parentQueue, autoCreatedLeafQueue.getParent()); } + + @Test + @Timeout(value = 20) + public void testAutoCreatedLeafQueueRecoversCapacityAfterNodeRegisters() + throws Exception { + try { + CapacitySchedulerConfiguration csConf = + setupSimpleQueueConfiguration(false); + setupMinMaxResourceConfiguration(csConf); + csConf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class, + ResourceScheduler.class); + csConf.setOverrideWithQueueMappings(true); + setupGroupQueueMappings(QUEUED, csConf, "%user"); + + mockRM = new MockRM(csConf); + cs = (CapacityScheduler) mockRM.getResourceScheduler(); + mockRM.start(); + cs.start(); + + submitApp(mockRM, cs.getQueue(QUEUED), TEST_GROUPUSER, TEST_GROUPUSER, 1, + 1); + + AutoCreatedLeafQueue autoCreatedLeafQueue = + (AutoCreatedLeafQueue) cs.getQueue(TEST_GROUPUSER); + ManagedParentQueue parentQueue = (ManagedParentQueue) cs.getQueue(QUEUED); + assertNotNull(autoCreatedLeafQueue, "Auto Creation of Queue failed"); + assertEquals(parentQueue, autoCreatedLeafQueue.getParent()); + + GuaranteedOrZeroCapacityOverTimePolicy policy = + (GuaranteedOrZeroCapacityOverTimePolicy) parentQueue + .getAutoCreatedQueueManagementPolicy(); + + // With no cluster resource the queue must be created at zero capacity and + // must NOT be activated - activating it here is exactly what strands it at + // zero capacity for the lifetime of the queue. + assertEquals(0.0f, autoCreatedLeafQueue.getCapacity(), EPSILON); + assertEquals(0.0f, autoCreatedLeafQueue.getAbsoluteCapacity(), EPSILON); + assertFalse(policy.isActive(autoCreatedLeafQueue, NO_LABEL), + "Queue must not be activated while the cluster resource is zero"); + + // A NodeManager registers: the cluster resource becomes available. + mockRM.registerNode("127.0.0.1:1234", 250 * GB, 40); + mockRM.drainEvents(); + + // The queue management monitor runs, which is what + // QueueManagementDynamicEditPolicy does on its monitoring interval. + List changes = + policy.computeQueueManagementChanges(); + parentQueue.validateAndApplyQueueManagementChanges(changes); + + validateCapacities(autoCreatedLeafQueue, 0.4f, 0.04f, 1f, 0.6f); + assertTrue(policy.isActive(autoCreatedLeafQueue, NO_LABEL), + "Queue must be activated once the cluster resource is available"); + assertEquals(0.04f, + policy.getAbsoluteActivatedChildQueueCapacity(NO_LABEL), EPSILON); + } finally { + cleanupQueue(TEST_GROUPUSER); + } + } }