From 2e3f5e67fafead4686b376b308505b85c3daf533 Mon Sep 17 00:00:00 2001 From: slfan1989 Date: Sat, 1 Feb 2025 18:01:38 +0800 Subject: [PATCH 1/2] MAPREDUCE-7497. mapreduce tests have stopped running. --- .../hadoop-mapreduce-client-app/pom.xml | 5 +++++ .../mapreduce/v2/app/webapp/TestAMWebServices.java | 11 ++++++----- .../hadoop-mapreduce-client-core/pom.xml | 5 +++++ .../hadoop-mapreduce-client-hs/pom.xml | 5 +++++ .../mapreduce/v2/hs/webapp/TestHsWebServices.java | 13 +++++++------ .../hadoop-mapreduce-client-jobclient/pom.xml | 5 +++++ .../hadoop-mapreduce-client-nativetask/pom.xml | 5 +++++ .../hadoop-mapreduce-client-shuffle/pom.xml | 5 +++++ hadoop-project/pom.xml | 4 ++-- .../records/timelineservice/ApplicationEntity.java | 7 ++++++- 10 files changed, 51 insertions(+), 14 deletions(-) diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/pom.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/pom.xml index 4bd8ca970e525d..b54e96d213a01d 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/pom.xml +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/pom.xml @@ -168,6 +168,11 @@ junit-platform-launcher test + + org.junit.vintage + junit-vintage-engine + test + diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebServices.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebServices.java index 2cc59ed6642669..560bc751560993 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebServices.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebServices.java @@ -28,13 +28,14 @@ import java.util.Set; import javax.ws.rs.core.MediaType; +import javax.ws.rs.NotAcceptableException; import javax.ws.rs.NotFoundException; -import javax.ws.rs.ServiceUnavailableException; import javax.ws.rs.client.WebTarget; import javax.ws.rs.core.Application; import javax.ws.rs.core.Response; + +import static javax.ws.rs.core.Response.Status.NOT_ACCEPTABLE; import static javax.ws.rs.core.Response.Status.NOT_FOUND; -import static javax.ws.rs.core.Response.Status.SERVICE_UNAVAILABLE; import javax.xml.parsers.DocumentBuilder; import javax.xml.parsers.DocumentBuilderFactory; @@ -231,15 +232,15 @@ public void testInvalidUri2() throws JSONException, Exception { @Test public void testInvalidAccept() throws JSONException, Exception { - WebTarget r = target(); + WebTarget r = targetWithJsonObject(); String responseStr = ""; try { responseStr = r.path("ws").path("v1").path("mapreduce") .request(MediaType.TEXT_PLAIN).get(String.class); fail("should have thrown exception on invalid uri"); - } catch (ServiceUnavailableException sue) { + } catch (NotAcceptableException sue) { Response response = sue.getResponse(); - assertResponseStatusCode(SERVICE_UNAVAILABLE, response.getStatusInfo()); + assertResponseStatusCode(NOT_ACCEPTABLE, response.getStatusInfo()); WebServicesTestUtils.checkStringMatch( "error string exists and shouldn't", "", responseStr); } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/pom.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/pom.xml index ef424e3e992eed..81457e6845ec37 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/pom.xml +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/pom.xml @@ -92,6 +92,11 @@ junit-platform-launcher test + + org.junit.vintage + junit-vintage-engine + test + diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/pom.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/pom.xml index d8c9eaf19bb203..57f954ecd3841c 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/pom.xml +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/pom.xml @@ -115,6 +115,11 @@ junit-platform-launcher test + + org.junit.vintage + junit-vintage-engine + test + diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServices.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServices.java index c34df08b7ff7c3..b320868242e29f 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServices.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServices.java @@ -20,13 +20,14 @@ import static org.apache.hadoop.yarn.webapp.WebServicesTestUtils.assertResponseStatusCode; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; import static org.mockito.Mockito.mock; import java.io.StringReader; import javax.servlet.http.HttpServletResponse; +import javax.ws.rs.NotAcceptableException; import javax.ws.rs.NotFoundException; -import javax.ws.rs.ServiceUnavailableException; import javax.ws.rs.client.WebTarget; import javax.ws.rs.core.Application; import javax.ws.rs.core.MediaType; @@ -232,12 +233,12 @@ public void testInvalidAccept() throws JSONException, Exception { WebTarget r = target(); String responseStr = ""; try { - Response response = - r.path("ws").path("v1").path("history").request(MediaType.TEXT_PLAIN).get(); - throw new ServiceUnavailableException(response); - } catch (ServiceUnavailableException ue) { + responseStr = r.path("ws").path("v1").path("history"). + request(MediaType.TEXT_PLAIN).get(String.class); + fail("should have thrown exception on invalid uri"); + } catch (NotAcceptableException ue) { Response response = ue.getResponse(); - assertResponseStatusCode(Response.Status.SERVICE_UNAVAILABLE, + assertResponseStatusCode(Response.Status.NOT_ACCEPTABLE, response.getStatusInfo()); WebServicesTestUtils.checkStringMatch( "error string exists and shouldn't", "", responseStr); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/pom.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/pom.xml index 8305867ba08273..b5deed6231e32c 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/pom.xml +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/pom.xml @@ -148,6 +148,11 @@ junit-platform-launcher test + + org.junit.vintage + junit-vintage-engine + test + diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/pom.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/pom.xml index 5ffa4f52b30808..f162b7037ee599 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/pom.xml +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/pom.xml @@ -80,6 +80,11 @@ junit-platform-launcher test + + org.junit.vintage + junit-vintage-engine + test + diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/pom.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/pom.xml index fee97311ee5895..b1c1b51c81fc9b 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/pom.xml +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/pom.xml @@ -66,6 +66,11 @@ junit-platform-launcher test + + org.junit.vintage + junit-vintage-engine + test + diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml index b51881d58faaaa..a4a9b94c863d00 100644 --- a/hadoop-project/pom.xml +++ b/hadoop-project/pom.xml @@ -185,8 +185,8 @@ --add-opens=java.base/sun.security.x509=ALL-UNNAMED - -Xmx2048m -XX:+HeapDumpOnOutOfMemoryError ${extraJavaTestArgs} - 3.0.0-M1 + -Xmx2048m -Xss2m -XX:+HeapDumpOnOutOfMemoryError ${extraJavaTestArgs} + 3.0.0-M4 ${maven-surefire-plugin.version} ${maven-surefire-plugin.version} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/ApplicationEntity.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/ApplicationEntity.java index 20226aa8e9c682..58f65f34e1f1fc 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/ApplicationEntity.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/ApplicationEntity.java @@ -43,7 +43,12 @@ public ApplicationEntity(TimelineEntity entity) { } public String getQueue() { - return getInfo().get(QUEUE_INFO_KEY).toString(); + if (getInfo() != null) { + if (getInfo().containsKey(QUEUE_INFO_KEY)) { + return getInfo().get(QUEUE_INFO_KEY).toString(); + } + } + return ""; } public void setQueue(String queue) { From a5b718ebdbd843271735f7037bf7a180d8ce6e13 Mon Sep 17 00:00:00 2001 From: fanshilun Date: Mon, 3 Feb 2025 21:44:35 +0800 Subject: [PATCH 2/2] YARN-11764. Add the JsonInclude annotation. --- .../api/records/timelineservice/ApplicationEntity.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/ApplicationEntity.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/ApplicationEntity.java index 58f65f34e1f1fc..87f3d2e9328f52 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/ApplicationEntity.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/ApplicationEntity.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.yarn.api.records.timelineservice; +import com.fasterxml.jackson.annotation.JsonInclude; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; @@ -42,13 +43,12 @@ public ApplicationEntity(TimelineEntity entity) { } } + @JsonInclude(JsonInclude.Include.NON_NULL) public String getQueue() { - if (getInfo() != null) { - if (getInfo().containsKey(QUEUE_INFO_KEY)) { - return getInfo().get(QUEUE_INFO_KEY).toString(); - } + if (getInfo() != null && getInfo().containsKey(QUEUE_INFO_KEY)) { + return getInfo().get(QUEUE_INFO_KEY).toString(); } - return ""; + return null; } public void setQueue(String queue) {