RM planner test and framework changes#581

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
abhidotravi wants to merge 7 commits into
base:master
Choose a base branch
from
Open

RM planner test and framework changes #581

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
a38429f
RM planner test and framework changes
abhidotravi Apr 24, 2019
8f11f93
Add support for complex selector
abhidotravi Apr 24, 2019
6b07e4a
Add a run script to run tests
abhidotravi Apr 25, 2019
15a5fbd
Add help to run script
abhidotravi Apr 25, 2019
0d509c9
Minor cosmetic change
abhidotravi Apr 25, 2019
483ee23
More minor cosmetic changes
abhidotravi Apr 25, 2019
a566d6a
Add more tests for Planner and Selector
abhidotravi May 2, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 13 additions & 0 deletions bin/run_testng
View file
Open in desktop
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
#export MAVEN_OPTS=$MAVEN_OPTS" -agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=5006"
bin=`dirname "${BASH_SOURCE-$0}"`
bin=`cd "$bin">/dev/null; pwd`
cd ${bin}/../framework
source ../conf/drillTestConfig.properties
if [[ $1 == "-Dtest="* ]]; then
mvn test $*
elif [[ $1 == "-h" || $1 == "-help" ]]; then
echo "bin/runtestng [-Dtest=<class_name>[#<method_name>]]"
else
echo "Running the entire Suite"
mvn test -Ptestng $*
fi
69 changes: 55 additions & 14 deletions framework/src/main/java/org/apache/drill/test/framework/DrillQueryProfile.java
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -6,6 +6,7 @@
import oadd.org.apache.drill.exec.proto.UserBitShared;

import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;

@JsonIgnoreProperties(ignoreUnknown = true)
Expand DownExpand Up@@ -203,12 +204,12 @@ public String toString() {
}

/**
* Get optimal memory allocated per operator.
* Utility parses the DrillQueryProfile
* Get total optimal memory allocated (in bytes) for specified operator, across all drillbits.
*
* @param operator
* @return
*/
public long getOptimalMemoryPerOperator(final UserBitShared.CoreOperatorType operator) {
public long getTotalOptimalMemoryPerOperator(final UserBitShared.CoreOperatorType operator) {
return this.fragmentProfiles
.stream()
.flatMap(f -> f.minorFragmentProfiles
Expand All@@ -219,6 +220,57 @@ public long getOptimalMemoryPerOperator(final UserBitShared.CoreOperatorType ope
.sum();
}

/**
* Returns the max of optimal memory allocated (in bytes) to specified operator on a drillbit.
*
* @param operator
* @return
*/
public long getOptimalMemoryPerOperatorPerNode(final UserBitShared.CoreOperatorType operator) {
return this.fragmentProfiles
.stream()
.flatMap(f -> f.minorFragmentProfiles
.stream())
.collect(Collectors.groupingBy(m -> m.endpoint.address))
.entrySet()
.stream()
.collect(Collectors.toMap(Map.Entry::getKey,
e -> e.getValue()
.stream()
.flatMap(m -> m.operatorProfiles
.stream()
.filter(o -> o.operatorId == operator.getNumber()))
.mapToLong(o -> o.optimalMemAllocation)
.sum()))
.entrySet()
.stream()
.mapToLong(Map.Entry::getValue)
.max()
.orElse(0);
}

/**
* Total optimal memory required (in bytes) for the query.
* @return total optimal memory required for the query (as estimated by the RM planner).
*/
public long getTotalOptimalMemory() {
return getOperatorsFromProfile()
.stream()
.mapToLong(this::getTotalOptimalMemoryPerOperator)
.sum();
}

/**
* Returns the maximum of estimated optimal memory (in bytes) required on a drillbit.
* @return total optimal memory required for the query (as estimated by the RM planner).
*/
public long getTotalOptimalMemoryPerNode() {
return getOperatorsFromProfile()
.stream()
.mapToLong(this::getOptimalMemoryPerOperatorPerNode)
.sum();
}

/**
* Get different operators in the profile.
* @return a list of operators in the query profile.
Expand All@@ -233,17 +285,6 @@ public List<UserBitShared.CoreOperatorType> getOperatorsFromProfile() {
.mapToObj(UserBitShared.CoreOperatorType::forNumber)
.collect(Collectors.toList());
}

/**
* Total optimal memory required for the query.
* @return total optimal memory required for the query (as estimated by the RM planner).
*/
public long getTotalOptimalMemoryEstimate() {
return getOperatorsFromProfile()
.stream()
.mapToLong(this::getOptimalMemoryPerOperator)
.sum();
}
}


29 changes: 24 additions & 5 deletions framework/src/main/java/org/apache/drill/test/framework/DrillRMConfig.java
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -36,6 +36,9 @@ public class DrillRMConfig implements DrillConfigRenderer {
//Selector Configurations
public static final String SELECTOR_TAG_KEY = "tag";
public static final String SELECTOR_ACL_KEY = "acl";
public static final String SELECTOR_OR_KEY = "or";
public static final String SELECTOR_AND_KEY = "and";
public static final String SELECTOR_NOT_EQUAL_KEY = "not_equal";

//ACL Configurations
public static final String ACL_USERS_KEY = "users";
Expand DownExpand Up@@ -74,8 +77,10 @@ public class DrillRMConfig implements DrillConfigRenderer {
public static class SelectorConfig implements DrillConfigRenderer {

public String tag;

public AclConfig acl;
public SelectorConfig not_equal;
public List<SelectorConfig> or;
public List<SelectorConfig> and;

@Override
public String render() {
Expand All@@ -98,6 +103,21 @@ public String render(final int acc) {
sb.append(formatConfig(nextAcc, SELECTOR_ACL_KEY, acl));
}

if (not_equal != null) {
ensureAtleastOneField = true;
sb.append(formatConfig(nextAcc, SELECTOR_NOT_EQUAL_KEY, not_equal));
}

if (or != null) {
ensureAtleastOneField = true;
sb.append(formatConfig(nextAcc, SELECTOR_OR_KEY, or));
}

if (and != null) {
ensureAtleastOneField = true;
sb.append(formatConfig(nextAcc, SELECTOR_AND_KEY, and));
}

if(ensureAtleastOneField) {
sb.deleteCharAt(sb.length() - 1)
.deleteCharAt(sb.length() - 1)
Expand All@@ -119,7 +139,6 @@ public String render(final int acc) {
public static class AclConfig implements DrillConfigRenderer {

public List<String> users;

public List<String> groups;

@Override
Expand DownExpand Up@@ -163,7 +182,7 @@ public String render(final int acc) {
public static class QueueConfig implements DrillConfigRenderer {

@JsonProperty(QUEUE_MAX_QUERY_MEMORY_PER_NODE_KEY)
public long maxQueryMemoryPerNodeInMB;
public long maxQueryMemoryPerNode; //in bytes

@JsonProperty(QUEUE_MAX_WAITING_KEY)
public int maxWaitingQueries;
Expand All@@ -185,9 +204,9 @@ public String render(final int acc) {
StringBuilder sb = new StringBuilder("{\n");
final int nextAcc = acc+2;

if (maxQueryMemoryPerNodeInMB > 0) {
if (maxQueryMemoryPerNode > 0) {
ensureAtleastOneField = true;
sb.append(formatConfig(nextAcc, QUEUE_MAX_QUERY_MEMORY_PER_NODE_KEY, maxQueryMemoryPerNodeInMB));
sb.append(formatConfig(nextAcc, QUEUE_MAX_QUERY_MEMORY_PER_NODE_KEY, maxQueryMemoryPerNode));
}

if (maxWaitingQueries > 0) {
Expand Down
30 changes: 24 additions & 6 deletions framework/src/test/java/org/apache/drill/test/framework/DrillTestFrameworkUnitTests.java
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -21,8 +21,9 @@
import java.util.Properties;

import static org.apache.drill.test.framework.DrillTestDefaults.DRILL_EXEC_RM_CONFIG_KEY;
import static org.apache.drill.test.framework.common.DrillTestNGDefaults.BASIC_RM_CONFIG_FILEPATH;
import static org.apache.drill.test.framework.common.DrillTestNGDefaults.UNIT_GROUP;
import static org.apache.drill.test.framework.common.DrillTestNGDefaults.SAMPLE_RM_CONFIG_NAME;
import static org.apache.drill.test.framework.common.DrillTestNGDefaults.PROD_RM_CONFIG_FILEPATH;

@Test(groups = UNIT_GROUP)
public class DrillTestFrameworkUnitTests extends DrillJavaTestBase {
Expand DownExpand Up@@ -84,7 +85,7 @@ public void testQueryProfileDoesNotExist() {
@Test(groups = UNIT_GROUP)
public void testReadSampleRMConfigFile() {
try {
DrillRMConfig drillRMConfig = DrillRMConfig.load(SAMPLE_RM_CONFIG_NAME);
DrillRMConfig drillRMConfig = DrillRMConfig.load(PROD_RM_CONFIG_FILEPATH);
Assert.assertEquals(drillRMConfig.poolName, "root",
"Root resource pool name did not match");

Expand All@@ -96,6 +97,22 @@ public void testReadSampleRMConfigFile() {
}
}

/**
* Test reading a sample RM config file, with complex selectors, in to a Java Bean.
*/
@Test(groups = UNIT_GROUP)
public void testReadComplexSelectorsRMConfigFile() {
try {
DrillRMConfig drillRMConfig = DrillRMConfig.load(BASIC_RM_CONFIG_FILEPATH);
Assert.assertEquals(drillRMConfig.childPools.get(1).selector.or.size(), 2,
"Or selector should have had 2 children!");

} catch (Exception e) {
e.printStackTrace();
Assert.fail(e.getMessage());
}
}

/**
* Negative test to validate the behavior when the config file does not exist.
*/
Expand All@@ -112,7 +129,7 @@ public void testLoadConfigWhenFileDoesNotExist() throws IOException {
@Test(groups = UNIT_GROUP)
public void testConfigFileRenderer() {
try {
DrillRMConfig drillRMConfig = DrillRMConfig.load(SAMPLE_RM_CONFIG_NAME);
DrillRMConfig drillRMConfig = DrillRMConfig.load(PROD_RM_CONFIG_FILEPATH);
Assert.assertEquals(drillRMConfig.poolName, "root",
"Root resource pool name did not match");

Expand DownExpand Up@@ -149,7 +166,7 @@ public void testWriteRMConfigToFile() throws IOException {
}

try (BufferedWriter writer = new BufferedWriter(new FileWriter(filePath))) {
DrillRMConfig drillRMConfig = DrillRMConfig.load(SAMPLE_RM_CONFIG_NAME);
DrillRMConfig drillRMConfig = DrillRMConfig.load(PROD_RM_CONFIG_FILEPATH);
writer.write(DRILL_EXEC_RM_CONFIG_KEY + ":" + drillRMConfig.render());
}

Expand DownExpand Up@@ -178,14 +195,15 @@ public void testTotalMemoryForQueryProfile() {
DrillQueryProfile profile = Utils.getQueryProfile(queryId);
Assert.assertEquals(profile.queryId, queryId);

long rmMemEstimate = profile.getTotalOptimalMemoryEstimate();
long rmMemEstimate = profile.getTotalOptimalMemoryPerNode();
LOG.info("Memory estimated by RM planner: " + rmMemEstimate);
Assert.assertTrue(rmMemEstimate > 0,
"RM estimated memory should be greater than 0");
List<UserBitShared.CoreOperatorType> operators = profile.getOperatorsFromProfile();
Assert.assertTrue(operators.size() > 0,
"Number of operators in the profile should be greater than 0");
operators.forEach(LOG::info);
operators.forEach(o -> LOG.info("Operator: " + o + ", Optimal Memory per Node in bytes: " +
profile.getOptimalMemoryPerOperatorPerNode(o)));
} catch (Exception e) {
e.printStackTrace();
Assert.fail(e.getMessage());
Expand Down
11 changes: 7 additions & 4 deletions framework/src/test/java/org/apache/drill/test/framework/common/DrillTestNGDefaults.java
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -5,13 +5,16 @@
public final class DrillTestNGDefaults {
public static final String FUNCTIONAL_GROUP = "functional";
public static final String UNIT_GROUP = "unit";
public static final String SAMPLE_RM_CONFIG_NAME =
DrillTestDefaults.CWD + "/src/test/resources/sample-drill-rm-override.conf";
public static final String BASIC_RM_CONFIG_NAME =
DrillTestDefaults.CWD + "/src/test/resources/basic-drill-rm-override.conf";
public static final String PROD_RM_CONFIG_FILEPATH =
DrillTestDefaults.CWD + "/src/test/resources/prod-rm.conf";
public static final String BASIC_RM_CONFIG_FILEPATH =
DrillTestDefaults.CWD + "/src/test/resources/basic-rm.conf";
public static final String DEFAULT_RM_CONFIG_FILEPATH =
DrillTestDefaults.CWD + "/src/test/resources/default-rm.conf";

public static String CONNECTION_URL_FOR_DRILLBIT(final String hostnameOrIp) {
return String.format("jdbc:drill:drillbit=%s", hostnameOrIp);
}
public static final String NO_RESOURCE_POOL_ERROR = "No resource pools to choose from for the query";
public static final String TPCH_01_PARQUET_SCHEMA = "dfs.drilltestdirtpch01parquet";
}
Loading
, 'i'); if (__m === '*' || __re.test(location.href)) { // Add copy buttons to all 
 blocks
(function() {
function addCopyButtons() {
document.querySelectorAll('pre code').forEach(function(codeBlock) {
if (codeBlock.parentElement.hasAttribute('data-copy-added')) return;
codeBlock.parentElement.setAttribute('data-copy-added', 'true');
var btn = document.createElement('button');
btn.textContent = 'Copy';
btn.style.cssText = 'position:absolute;top:4px;right:4px;padding:2px 8px;font-size:11px;background:#4ecdc4;border:none;border-radius:4px;color:#1a1a2e;cursor:pointer;opacity:0.7;transition:opacity 0.2s;';
btn.onmouseover = function() { this.style.opacity = '1'; };
btn.onmouseout = function() { this.style.opacity = '0.7'; };
btn.onclick = function() {
navigator.clipboard.writeText(codeBlock.textContent).then(function() {
btn.textContent = 'Copied!';
setTimeout(function() { btn.textContent = 'Copy'; }, 1500);
});
};
codeBlock.parentElement.style.position = 'relative';
codeBlock.parentElement.appendChild(btn);
});
}
addCopyButtons();
// Re-run on dynamic content
var observer = new MutationObserver(addCopyButtons);
observer.observe(document.body, { childList: true, subtree: true });
})();
}
} catch(__e) { console.warn('[Userscript:Add Copy Buttons to Code Blocks]', __e); }
})();
(function(){
try {
var __m = "github.com";
var __re = new RegExp('^' + "github\\.com" + '

Skip to content
RM planner test and framework changes#581

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Open

abhidotravi
wants to merge
7
commits into

base:master
Choose a base branch

from

 Open


RM planner test and framework changes
#581
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


7 commits


Select commit
Hold shift + click to select a range
a38429f

RM planner test and framework changes


abhidotravi Apr 24, 2019
8f11f93

Add support for complex selector


abhidotravi Apr 24, 2019
6b07e4a

Add a run script to run tests


abhidotravi Apr 25, 2019
15a5fbd

Add help to run script


abhidotravi Apr 25, 2019
0d509c9

Minor cosmetic change


abhidotravi Apr 25, 2019
483ee23

More minor cosmetic changes


abhidotravi Apr 25, 2019
a566d6a

Add more tests for Planner and Selector


abhidotravi May 2, 2019

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


13 changes: 13 additions & 0 deletions
bin/run_testng

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
#export MAVEN_OPTS=$MAVEN_OPTS" -agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=5006"
bin=`dirname "${BASH_SOURCE-$0}"`
bin=`cd "$bin">/dev/null; pwd`
cd ${bin}/../framework
source ../conf/drillTestConfig.properties
if [[ $1 == "-Dtest="* ]]; then
 mvn test $*
elif [[ $1 == "-h" || $1 == "-help" ]]; then
 echo "bin/runtestng [-Dtest=<class_name>[#<method_name>]]"
else
 echo "Running the entire Suite"
 mvn test -Ptestng $*
fi

69 changes: 55 additions & 14 deletions
framework/src/main/java/org/apache/drill/test/framework/DrillQueryProfile.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -6,6 +6,7 @@
import oadd.org.apache.drill.exec.proto.UserBitShared;

import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;

@JsonIgnoreProperties(ignoreUnknown = true)
Expand DownExpand Up@@ -203,12 +204,12 @@ public String toString() {
 }

 /**
 * Get optimal memory allocated per operator.
 * Utility parses the DrillQueryProfile
 * Get total optimal memory allocated (in bytes) for specified operator, across all drillbits.
 *
 * @param operator
 * @return
 */
 public long getOptimalMemoryPerOperator(final UserBitShared.CoreOperatorType operator) {
 public long getTotalOptimalMemoryPerOperator(final UserBitShared.CoreOperatorType operator) {
 return this.fragmentProfiles
 .stream()
 .flatMap(f -> f.minorFragmentProfiles
Expand All@@ -219,6 +220,57 @@ public long getOptimalMemoryPerOperator(final UserBitShared.CoreOperatorType ope
 .sum();
 }

 /**
 * Returns the max of optimal memory allocated (in bytes) to specified operator on a drillbit.
 *
 * @param operator
 * @return
 */
 public long getOptimalMemoryPerOperatorPerNode(final UserBitShared.CoreOperatorType operator) {
 return this.fragmentProfiles
 .stream()
 .flatMap(f -> f.minorFragmentProfiles
 .stream())
 .collect(Collectors.groupingBy(m -> m.endpoint.address))
 .entrySet()
 .stream()
 .collect(Collectors.toMap(Map.Entry::getKey,
 e -> e.getValue()
 .stream()
 .flatMap(m -> m.operatorProfiles
 .stream()
 .filter(o -> o.operatorId == operator.getNumber()))
 .mapToLong(o -> o.optimalMemAllocation)
 .sum()))
 .entrySet()
 .stream()
 .mapToLong(Map.Entry::getValue)
 .max()
 .orElse(0);
 }

 /**
 * Total optimal memory required (in bytes) for the query.
 * @return total optimal memory required for the query (as estimated by the RM planner).
 */
 public long getTotalOptimalMemory() {
 return getOperatorsFromProfile()
 .stream()
 .mapToLong(this::getTotalOptimalMemoryPerOperator)
 .sum();
 }

 /**
 * Returns the maximum of estimated optimal memory (in bytes) required on a drillbit.
 * @return total optimal memory required for the query (as estimated by the RM planner).
 */
 public long getTotalOptimalMemoryPerNode() {
 return getOperatorsFromProfile()
 .stream()
 .mapToLong(this::getOptimalMemoryPerOperatorPerNode)
 .sum();
 }

 /**
 * Get different operators in the profile.
 * @return a list of operators in the query profile.
Expand All@@ -233,17 +285,6 @@ public List<UserBitShared.CoreOperatorType> getOperatorsFromProfile() {
 .mapToObj(UserBitShared.CoreOperatorType::forNumber)
 .collect(Collectors.toList());
 }

 /**
 * Total optimal memory required for the query.
 * @return total optimal memory required for the query (as estimated by the RM planner).
 */
 public long getTotalOptimalMemoryEstimate() {
 return getOperatorsFromProfile()
 .stream()
 .mapToLong(this::getOptimalMemoryPerOperator)
 .sum();
 }
}



29 changes: 24 additions & 5 deletions
framework/src/main/java/org/apache/drill/test/framework/DrillRMConfig.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -36,6 +36,9 @@ public class DrillRMConfig implements DrillConfigRenderer {
 //Selector Configurations
 public static final String SELECTOR_TAG_KEY = "tag";
 public static final String SELECTOR_ACL_KEY = "acl";
 public static final String SELECTOR_OR_KEY = "or";
 public static final String SELECTOR_AND_KEY = "and";
 public static final String SELECTOR_NOT_EQUAL_KEY = "not_equal";

 //ACL Configurations
 public static final String ACL_USERS_KEY = "users";
Expand DownExpand Up@@ -74,8 +77,10 @@ public class DrillRMConfig implements DrillConfigRenderer {
 public static class SelectorConfig implements DrillConfigRenderer {

 public String tag;

 public AclConfig acl;
 public SelectorConfig not_equal;
 public List<SelectorConfig> or;
 public List<SelectorConfig> and;

 @Override
 public String render() {
Expand All@@ -98,6 +103,21 @@ public String render(final int acc) {
 sb.append(formatConfig(nextAcc, SELECTOR_ACL_KEY, acl));
 }

 if (not_equal != null) {
 ensureAtleastOneField = true;
 sb.append(formatConfig(nextAcc, SELECTOR_NOT_EQUAL_KEY, not_equal));
 }

 if (or != null) {
 ensureAtleastOneField = true;
 sb.append(formatConfig(nextAcc, SELECTOR_OR_KEY, or));
 }

 if (and != null) {
 ensureAtleastOneField = true;
 sb.append(formatConfig(nextAcc, SELECTOR_AND_KEY, and));
 }

 if(ensureAtleastOneField) {
 sb.deleteCharAt(sb.length() - 1)
 .deleteCharAt(sb.length() - 1)
Expand All@@ -119,7 +139,6 @@ public String render(final int acc) {
 public static class AclConfig implements DrillConfigRenderer {

 public List<String> users;

 public List<String> groups;

 @Override
Expand DownExpand Up@@ -163,7 +182,7 @@ public String render(final int acc) {
 public static class QueueConfig implements DrillConfigRenderer {

 @JsonProperty(QUEUE_MAX_QUERY_MEMORY_PER_NODE_KEY)
 public long maxQueryMemoryPerNodeInMB;
 public long maxQueryMemoryPerNode; //in bytes

 @JsonProperty(QUEUE_MAX_WAITING_KEY)
 public int maxWaitingQueries;
Expand All@@ -185,9 +204,9 @@ public String render(final int acc) {
 StringBuilder sb = new StringBuilder("{\n");
 final int nextAcc = acc+2;

 if (maxQueryMemoryPerNodeInMB > 0) {
 if (maxQueryMemoryPerNode > 0) {
 ensureAtleastOneField = true;
 sb.append(formatConfig(nextAcc, QUEUE_MAX_QUERY_MEMORY_PER_NODE_KEY, maxQueryMemoryPerNodeInMB));
 sb.append(formatConfig(nextAcc, QUEUE_MAX_QUERY_MEMORY_PER_NODE_KEY, maxQueryMemoryPerNode));
 }

 if (maxWaitingQueries > 0) {
Expand Down

30 changes: 24 additions & 6 deletions
framework/src/test/java/org/apache/drill/test/framework/DrillTestFrameworkUnitTests.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -21,8 +21,9 @@
import java.util.Properties;

import static org.apache.drill.test.framework.DrillTestDefaults.DRILL_EXEC_RM_CONFIG_KEY;
import static org.apache.drill.test.framework.common.DrillTestNGDefaults.BASIC_RM_CONFIG_FILEPATH;
import static org.apache.drill.test.framework.common.DrillTestNGDefaults.UNIT_GROUP;
import static org.apache.drill.test.framework.common.DrillTestNGDefaults.SAMPLE_RM_CONFIG_NAME;
import static org.apache.drill.test.framework.common.DrillTestNGDefaults.PROD_RM_CONFIG_FILEPATH;

@Test(groups = UNIT_GROUP)
public class DrillTestFrameworkUnitTests extends DrillJavaTestBase {
Expand DownExpand Up@@ -84,7 +85,7 @@ public void testQueryProfileDoesNotExist() {
 @Test(groups = UNIT_GROUP)
 public void testReadSampleRMConfigFile() {
 try {
 DrillRMConfig drillRMConfig = DrillRMConfig.load(SAMPLE_RM_CONFIG_NAME);
 DrillRMConfig drillRMConfig = DrillRMConfig.load(PROD_RM_CONFIG_FILEPATH);
 Assert.assertEquals(drillRMConfig.poolName, "root",
 "Root resource pool name did not match");

Expand All@@ -96,6 +97,22 @@ public void testReadSampleRMConfigFile() {
 }
 }

 /**
 * Test reading a sample RM config file, with complex selectors, in to a Java Bean.
 */
 @Test(groups = UNIT_GROUP)
 public void testReadComplexSelectorsRMConfigFile() {
 try {
 DrillRMConfig drillRMConfig = DrillRMConfig.load(BASIC_RM_CONFIG_FILEPATH);
 Assert.assertEquals(drillRMConfig.childPools.get(1).selector.or.size(), 2,
 "Or selector should have had 2 children!");

 } catch (Exception e) {
 e.printStackTrace();
 Assert.fail(e.getMessage());
 }
 }

 /**
 * Negative test to validate the behavior when the config file does not exist.
 */
Expand All@@ -112,7 +129,7 @@ public void testLoadConfigWhenFileDoesNotExist() throws IOException {
 @Test(groups = UNIT_GROUP)
 public void testConfigFileRenderer() {
 try {
 DrillRMConfig drillRMConfig = DrillRMConfig.load(SAMPLE_RM_CONFIG_NAME);
 DrillRMConfig drillRMConfig = DrillRMConfig.load(PROD_RM_CONFIG_FILEPATH);
 Assert.assertEquals(drillRMConfig.poolName, "root",
 "Root resource pool name did not match");

Expand DownExpand Up@@ -149,7 +166,7 @@ public void testWriteRMConfigToFile() throws IOException {
 }

 try (BufferedWriter writer = new BufferedWriter(new FileWriter(filePath))) {
 DrillRMConfig drillRMConfig = DrillRMConfig.load(SAMPLE_RM_CONFIG_NAME);
 DrillRMConfig drillRMConfig = DrillRMConfig.load(PROD_RM_CONFIG_FILEPATH);
 writer.write(DRILL_EXEC_RM_CONFIG_KEY + ":" + drillRMConfig.render());
 }

Expand DownExpand Up@@ -178,14 +195,15 @@ public void testTotalMemoryForQueryProfile() {
 DrillQueryProfile profile = Utils.getQueryProfile(queryId);
 Assert.assertEquals(profile.queryId, queryId);

 long rmMemEstimate = profile.getTotalOptimalMemoryEstimate();
 long rmMemEstimate = profile.getTotalOptimalMemoryPerNode();
 LOG.info("Memory estimated by RM planner: " + rmMemEstimate);
 Assert.assertTrue(rmMemEstimate > 0,
 "RM estimated memory should be greater than 0");
 List<UserBitShared.CoreOperatorType> operators = profile.getOperatorsFromProfile();
 Assert.assertTrue(operators.size() > 0,
 "Number of operators in the profile should be greater than 0");
 operators.forEach(LOG::info);
 operators.forEach(o -> LOG.info("Operator: " + o + ", Optimal Memory per Node in bytes: " +
 profile.getOptimalMemoryPerOperatorPerNode(o)));
 } catch (Exception e) {
 e.printStackTrace();
 Assert.fail(e.getMessage());
Expand Down

11 changes: 7 additions & 4 deletions
framework/src/test/java/org/apache/drill/test/framework/common/DrillTestNGDefaults.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -5,13 +5,16 @@
public final class DrillTestNGDefaults {
 public static final String FUNCTIONAL_GROUP = "functional";
 public static final String UNIT_GROUP = "unit";
 public static final String SAMPLE_RM_CONFIG_NAME =
 DrillTestDefaults.CWD + "/src/test/resources/sample-drill-rm-override.conf";
 public static final String BASIC_RM_CONFIG_NAME =
 DrillTestDefaults.CWD + "/src/test/resources/basic-drill-rm-override.conf";
 public static final String PROD_RM_CONFIG_FILEPATH =
 DrillTestDefaults.CWD + "/src/test/resources/prod-rm.conf";
 public static final String BASIC_RM_CONFIG_FILEPATH =
 DrillTestDefaults.CWD + "/src/test/resources/basic-rm.conf";
 public static final String DEFAULT_RM_CONFIG_FILEPATH =
 DrillTestDefaults.CWD + "/src/test/resources/default-rm.conf";

 public static String CONNECTION_URL_FOR_DRILLBIT(final String hostnameOrIp) {
 return String.format("jdbc:drill:drillbit=%s", hostnameOrIp);
 }
 public static final String NO_RESOURCE_POOL_ERROR = "No resource pools to choose from for the query";
 public static final String TPCH_01_PARQUET_SCHEMA = "dfs.drilltestdirtpch01parquet";
}
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
// Force GitHub README to respect dark mode
(function() {
var style = document.createElement('style');
style.textContent = '
.markdown-body {
color-scheme: dark light;
}
.markdown-body pre { background: #161b22 !important; }
.markdown-body code { background: rgba(110, 118, 129, 0.4) !important; }
.markdown-body table th, .markdown-body table td { border-color: #30363d !important; }
.markdown-body img { background: #0d1117; }
.markdown-body blockquote { border-left-color: #8b949e; }
.markdown-body hr { border-color: #30363d; }
';
document.head.appendChild(style);
})();
}
} catch(__e) { console.warn('[Userscript:GitHub Dark Mode README Fix]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
RM planner test and framework changes#581

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Open

abhidotravi
wants to merge
7
commits into

base:master
Choose a base branch

from

 Open


RM planner test and framework changes
#581
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


7 commits


Select commit
Hold shift + click to select a range
a38429f

RM planner test and framework changes


abhidotravi Apr 24, 2019
8f11f93

Add support for complex selector


abhidotravi Apr 24, 2019
6b07e4a

Add a run script to run tests


abhidotravi Apr 25, 2019
15a5fbd

Add help to run script


abhidotravi Apr 25, 2019
0d509c9

Minor cosmetic change


abhidotravi Apr 25, 2019
483ee23

More minor cosmetic changes


abhidotravi Apr 25, 2019
a566d6a

Add more tests for Planner and Selector


abhidotravi May 2, 2019

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


13 changes: 13 additions & 0 deletions
bin/run_testng

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
#export MAVEN_OPTS=$MAVEN_OPTS" -agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=5006"
bin=`dirname "${BASH_SOURCE-$0}"`
bin=`cd "$bin">/dev/null; pwd`
cd ${bin}/../framework
source ../conf/drillTestConfig.properties
if [[ $1 == "-Dtest="* ]]; then
 mvn test $*
elif [[ $1 == "-h" || $1 == "-help" ]]; then
 echo "bin/runtestng [-Dtest=<class_name>[#<method_name>]]"
else
 echo "Running the entire Suite"
 mvn test -Ptestng $*
fi

69 changes: 55 additions & 14 deletions
framework/src/main/java/org/apache/drill/test/framework/DrillQueryProfile.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -6,6 +6,7 @@
import oadd.org.apache.drill.exec.proto.UserBitShared;

import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;

@JsonIgnoreProperties(ignoreUnknown = true)
Expand DownExpand Up@@ -203,12 +204,12 @@ public String toString() {
 }

 /**
 * Get optimal memory allocated per operator.
 * Utility parses the DrillQueryProfile
 * Get total optimal memory allocated (in bytes) for specified operator, across all drillbits.
 *
 * @param operator
 * @return
 */
 public long getOptimalMemoryPerOperator(final UserBitShared.CoreOperatorType operator) {
 public long getTotalOptimalMemoryPerOperator(final UserBitShared.CoreOperatorType operator) {
 return this.fragmentProfiles
 .stream()
 .flatMap(f -> f.minorFragmentProfiles
Expand All@@ -219,6 +220,57 @@ public long getOptimalMemoryPerOperator(final UserBitShared.CoreOperatorType ope
 .sum();
 }

 /**
 * Returns the max of optimal memory allocated (in bytes) to specified operator on a drillbit.
 *
 * @param operator
 * @return
 */
 public long getOptimalMemoryPerOperatorPerNode(final UserBitShared.CoreOperatorType operator) {
 return this.fragmentProfiles
 .stream()
 .flatMap(f -> f.minorFragmentProfiles
 .stream())
 .collect(Collectors.groupingBy(m -> m.endpoint.address))
 .entrySet()
 .stream()
 .collect(Collectors.toMap(Map.Entry::getKey,
 e -> e.getValue()
 .stream()
 .flatMap(m -> m.operatorProfiles
 .stream()
 .filter(o -> o.operatorId == operator.getNumber()))
 .mapToLong(o -> o.optimalMemAllocation)
 .sum()))
 .entrySet()
 .stream()
 .mapToLong(Map.Entry::getValue)
 .max()
 .orElse(0);
 }

 /**
 * Total optimal memory required (in bytes) for the query.
 * @return total optimal memory required for the query (as estimated by the RM planner).
 */
 public long getTotalOptimalMemory() {
 return getOperatorsFromProfile()
 .stream()
 .mapToLong(this::getTotalOptimalMemoryPerOperator)
 .sum();
 }

 /**
 * Returns the maximum of estimated optimal memory (in bytes) required on a drillbit.
 * @return total optimal memory required for the query (as estimated by the RM planner).
 */
 public long getTotalOptimalMemoryPerNode() {
 return getOperatorsFromProfile()
 .stream()
 .mapToLong(this::getOptimalMemoryPerOperatorPerNode)
 .sum();
 }

 /**
 * Get different operators in the profile.
 * @return a list of operators in the query profile.
Expand All@@ -233,17 +285,6 @@ public List<UserBitShared.CoreOperatorType> getOperatorsFromProfile() {
 .mapToObj(UserBitShared.CoreOperatorType::forNumber)
 .collect(Collectors.toList());
 }

 /**
 * Total optimal memory required for the query.
 * @return total optimal memory required for the query (as estimated by the RM planner).
 */
 public long getTotalOptimalMemoryEstimate() {
 return getOperatorsFromProfile()
 .stream()
 .mapToLong(this::getOptimalMemoryPerOperator)
 .sum();
 }
}



29 changes: 24 additions & 5 deletions
framework/src/main/java/org/apache/drill/test/framework/DrillRMConfig.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -36,6 +36,9 @@ public class DrillRMConfig implements DrillConfigRenderer {
 //Selector Configurations
 public static final String SELECTOR_TAG_KEY = "tag";
 public static final String SELECTOR_ACL_KEY = "acl";
 public static final String SELECTOR_OR_KEY = "or";
 public static final String SELECTOR_AND_KEY = "and";
 public static final String SELECTOR_NOT_EQUAL_KEY = "not_equal";

 //ACL Configurations
 public static final String ACL_USERS_KEY = "users";
Expand DownExpand Up@@ -74,8 +77,10 @@ public class DrillRMConfig implements DrillConfigRenderer {
 public static class SelectorConfig implements DrillConfigRenderer {

 public String tag;

 public AclConfig acl;
 public SelectorConfig not_equal;
 public List<SelectorConfig> or;
 public List<SelectorConfig> and;

 @Override
 public String render() {
Expand All@@ -98,6 +103,21 @@ public String render(final int acc) {
 sb.append(formatConfig(nextAcc, SELECTOR_ACL_KEY, acl));
 }

 if (not_equal != null) {
 ensureAtleastOneField = true;
 sb.append(formatConfig(nextAcc, SELECTOR_NOT_EQUAL_KEY, not_equal));
 }

 if (or != null) {
 ensureAtleastOneField = true;
 sb.append(formatConfig(nextAcc, SELECTOR_OR_KEY, or));
 }

 if (and != null) {
 ensureAtleastOneField = true;
 sb.append(formatConfig(nextAcc, SELECTOR_AND_KEY, and));
 }

 if(ensureAtleastOneField) {
 sb.deleteCharAt(sb.length() - 1)
 .deleteCharAt(sb.length() - 1)
Expand All@@ -119,7 +139,6 @@ public String render(final int acc) {
 public static class AclConfig implements DrillConfigRenderer {

 public List<String> users;

 public List<String> groups;

 @Override
Expand DownExpand Up@@ -163,7 +182,7 @@ public String render(final int acc) {
 public static class QueueConfig implements DrillConfigRenderer {

 @JsonProperty(QUEUE_MAX_QUERY_MEMORY_PER_NODE_KEY)
 public long maxQueryMemoryPerNodeInMB;
 public long maxQueryMemoryPerNode; //in bytes

 @JsonProperty(QUEUE_MAX_WAITING_KEY)
 public int maxWaitingQueries;
Expand All@@ -185,9 +204,9 @@ public String render(final int acc) {
 StringBuilder sb = new StringBuilder("{\n");
 final int nextAcc = acc+2;

 if (maxQueryMemoryPerNodeInMB > 0) {
 if (maxQueryMemoryPerNode > 0) {
 ensureAtleastOneField = true;
 sb.append(formatConfig(nextAcc, QUEUE_MAX_QUERY_MEMORY_PER_NODE_KEY, maxQueryMemoryPerNodeInMB));
 sb.append(formatConfig(nextAcc, QUEUE_MAX_QUERY_MEMORY_PER_NODE_KEY, maxQueryMemoryPerNode));
 }

 if (maxWaitingQueries > 0) {
Expand Down

30 changes: 24 additions & 6 deletions
framework/src/test/java/org/apache/drill/test/framework/DrillTestFrameworkUnitTests.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -21,8 +21,9 @@
import java.util.Properties;

import static org.apache.drill.test.framework.DrillTestDefaults.DRILL_EXEC_RM_CONFIG_KEY;
import static org.apache.drill.test.framework.common.DrillTestNGDefaults.BASIC_RM_CONFIG_FILEPATH;
import static org.apache.drill.test.framework.common.DrillTestNGDefaults.UNIT_GROUP;
import static org.apache.drill.test.framework.common.DrillTestNGDefaults.SAMPLE_RM_CONFIG_NAME;
import static org.apache.drill.test.framework.common.DrillTestNGDefaults.PROD_RM_CONFIG_FILEPATH;

@Test(groups = UNIT_GROUP)
public class DrillTestFrameworkUnitTests extends DrillJavaTestBase {
Expand DownExpand Up@@ -84,7 +85,7 @@ public void testQueryProfileDoesNotExist() {
 @Test(groups = UNIT_GROUP)
 public void testReadSampleRMConfigFile() {
 try {
 DrillRMConfig drillRMConfig = DrillRMConfig.load(SAMPLE_RM_CONFIG_NAME);
 DrillRMConfig drillRMConfig = DrillRMConfig.load(PROD_RM_CONFIG_FILEPATH);
 Assert.assertEquals(drillRMConfig.poolName, "root",
 "Root resource pool name did not match");

Expand All@@ -96,6 +97,22 @@ public void testReadSampleRMConfigFile() {
 }
 }

 /**
 * Test reading a sample RM config file, with complex selectors, in to a Java Bean.
 */
 @Test(groups = UNIT_GROUP)
 public void testReadComplexSelectorsRMConfigFile() {
 try {
 DrillRMConfig drillRMConfig = DrillRMConfig.load(BASIC_RM_CONFIG_FILEPATH);
 Assert.assertEquals(drillRMConfig.childPools.get(1).selector.or.size(), 2,
 "Or selector should have had 2 children!");

 } catch (Exception e) {
 e.printStackTrace();
 Assert.fail(e.getMessage());
 }
 }

 /**
 * Negative test to validate the behavior when the config file does not exist.
 */
Expand All@@ -112,7 +129,7 @@ public void testLoadConfigWhenFileDoesNotExist() throws IOException {
 @Test(groups = UNIT_GROUP)
 public void testConfigFileRenderer() {
 try {
 DrillRMConfig drillRMConfig = DrillRMConfig.load(SAMPLE_RM_CONFIG_NAME);
 DrillRMConfig drillRMConfig = DrillRMConfig.load(PROD_RM_CONFIG_FILEPATH);
 Assert.assertEquals(drillRMConfig.poolName, "root",
 "Root resource pool name did not match");

Expand DownExpand Up@@ -149,7 +166,7 @@ public void testWriteRMConfigToFile() throws IOException {
 }

 try (BufferedWriter writer = new BufferedWriter(new FileWriter(filePath))) {
 DrillRMConfig drillRMConfig = DrillRMConfig.load(SAMPLE_RM_CONFIG_NAME);
 DrillRMConfig drillRMConfig = DrillRMConfig.load(PROD_RM_CONFIG_FILEPATH);
 writer.write(DRILL_EXEC_RM_CONFIG_KEY + ":" + drillRMConfig.render());
 }

Expand DownExpand Up@@ -178,14 +195,15 @@ public void testTotalMemoryForQueryProfile() {
 DrillQueryProfile profile = Utils.getQueryProfile(queryId);
 Assert.assertEquals(profile.queryId, queryId);

 long rmMemEstimate = profile.getTotalOptimalMemoryEstimate();
 long rmMemEstimate = profile.getTotalOptimalMemoryPerNode();
 LOG.info("Memory estimated by RM planner: " + rmMemEstimate);
 Assert.assertTrue(rmMemEstimate > 0,
 "RM estimated memory should be greater than 0");
 List<UserBitShared.CoreOperatorType> operators = profile.getOperatorsFromProfile();
 Assert.assertTrue(operators.size() > 0,
 "Number of operators in the profile should be greater than 0");
 operators.forEach(LOG::info);
 operators.forEach(o -> LOG.info("Operator: " + o + ", Optimal Memory per Node in bytes: " +
 profile.getOptimalMemoryPerOperatorPerNode(o)));
 } catch (Exception e) {
 e.printStackTrace();
 Assert.fail(e.getMessage());
Expand Down

11 changes: 7 additions & 4 deletions
framework/src/test/java/org/apache/drill/test/framework/common/DrillTestNGDefaults.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -5,13 +5,16 @@
public final class DrillTestNGDefaults {
 public static final String FUNCTIONAL_GROUP = "functional";
 public static final String UNIT_GROUP = "unit";
 public static final String SAMPLE_RM_CONFIG_NAME =
 DrillTestDefaults.CWD + "/src/test/resources/sample-drill-rm-override.conf";
 public static final String BASIC_RM_CONFIG_NAME =
 DrillTestDefaults.CWD + "/src/test/resources/basic-drill-rm-override.conf";
 public static final String PROD_RM_CONFIG_FILEPATH =
 DrillTestDefaults.CWD + "/src/test/resources/prod-rm.conf";
 public static final String BASIC_RM_CONFIG_FILEPATH =
 DrillTestDefaults.CWD + "/src/test/resources/basic-rm.conf";
 public static final String DEFAULT_RM_CONFIG_FILEPATH =
 DrillTestDefaults.CWD + "/src/test/resources/default-rm.conf";

 public static String CONNECTION_URL_FOR_DRILLBIT(final String hostnameOrIp) {
 return String.format("jdbc:drill:drillbit=%s", hostnameOrIp);
 }
 public static final String NO_RESOURCE_POOL_ERROR = "No resource pools to choose from for the query";
 public static final String TPCH_01_PARQUET_SCHEMA = "dfs.drilltestdirtpch01parquet";
}
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
// Highlight search terms from Google/DuckDuckGo/Bing referrer
(function() {
var ref = document.referrer;
var terms = [];
if (ref.includes('google.com') || ref.includes('duckduckgo.com') || ref.includes('bing.com')) {
var url = new URL(ref);
var q = url.searchParams.get('q') || url.searchParams.get('p');
if (q) {
terms = q.split(/\s+/).filter(function(t) { return t.length > 2; });
}
}
if (terms.length === 0) return;
var style = document.createElement('style');
style.textContent = '.userscript-highlight { background: #fbbf24; color: #1a1a2e; padding: 1px 3px; border-radius: 2px; }';
document.head.appendChild(style);
function highlight(node) {
if (node.nodeType === 3) { // text node
var text = node.textContent;
var found = false;
terms.forEach(function(term) {
var regex = new RegExp('(' + term.replace(/[.*+?^${}()|[\]\\]/g, '\\') + ')', 'gi');
if (regex.test(text)) {
found = true;
var frag = document.createDocumentFragment();
var parts = text.split(regex);
parts.forEach(function(part, i) {
if (i % 2 === 0) {
frag.appendChild(document.createTextNode(part));
} else {
var span = document.createElement('span');
span.className = 'userscript-highlight';
span.textContent = part;
frag.appendChild(span);
}
});
node.parentNode.replaceChild(frag, node);
}
});
} else if (node.nodeType === 1 && node.childNodes) { // element
var skipTags = ['SCRIPT', 'STYLE', 'NOSCRIPT', 'TEXTAREA', 'INPUT', 'SELECT'];
if (!skipTags.includes(node.tagName)) {
Array.from(node.childNodes).forEach(highlight);
}
}
}
highlight(document.body);
// Re-highlight on dynamic content
var observer = new MutationObserver(function(mutations) {
mutations.forEach(function(m) {
m.addedNodes.forEach(function(node) {
if (node.nodeType === 1 || node.nodeType === 3) highlight(node);
});
});
});
observer.observe(document.body, { childList: true, subtree: true });
})();
}
} catch(__e) { console.warn('[Userscript:Highlight Search Terms]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
RM planner test and framework changes#581

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Open

abhidotravi
wants to merge
7
commits into

base:master
Choose a base branch

from

 Open


RM planner test and framework changes
#581
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


7 commits


Select commit
Hold shift + click to select a range
a38429f

RM planner test and framework changes


abhidotravi Apr 24, 2019
8f11f93

Add support for complex selector


abhidotravi Apr 24, 2019
6b07e4a

Add a run script to run tests


abhidotravi Apr 25, 2019
15a5fbd

Add help to run script


abhidotravi Apr 25, 2019
0d509c9

Minor cosmetic change


abhidotravi Apr 25, 2019
483ee23

More minor cosmetic changes


abhidotravi Apr 25, 2019
a566d6a

Add more tests for Planner and Selector


abhidotravi May 2, 2019

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


13 changes: 13 additions & 0 deletions
bin/run_testng

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
#export MAVEN_OPTS=$MAVEN_OPTS" -agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=5006"
bin=`dirname "${BASH_SOURCE-$0}"`
bin=`cd "$bin">/dev/null; pwd`
cd ${bin}/../framework
source ../conf/drillTestConfig.properties
if [[ $1 == "-Dtest="* ]]; then
 mvn test $*
elif [[ $1 == "-h" || $1 == "-help" ]]; then
 echo "bin/runtestng [-Dtest=<class_name>[#<method_name>]]"
else
 echo "Running the entire Suite"
 mvn test -Ptestng $*
fi

69 changes: 55 additions & 14 deletions
framework/src/main/java/org/apache/drill/test/framework/DrillQueryProfile.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -6,6 +6,7 @@
import oadd.org.apache.drill.exec.proto.UserBitShared;

import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;

@JsonIgnoreProperties(ignoreUnknown = true)
Expand DownExpand Up@@ -203,12 +204,12 @@ public String toString() {
 }

 /**
 * Get optimal memory allocated per operator.
 * Utility parses the DrillQueryProfile
 * Get total optimal memory allocated (in bytes) for specified operator, across all drillbits.
 *
 * @param operator
 * @return
 */
 public long getOptimalMemoryPerOperator(final UserBitShared.CoreOperatorType operator) {
 public long getTotalOptimalMemoryPerOperator(final UserBitShared.CoreOperatorType operator) {
 return this.fragmentProfiles
 .stream()
 .flatMap(f -> f.minorFragmentProfiles
Expand All@@ -219,6 +220,57 @@ public long getOptimalMemoryPerOperator(final UserBitShared.CoreOperatorType ope
 .sum();
 }

 /**
 * Returns the max of optimal memory allocated (in bytes) to specified operator on a drillbit.
 *
 * @param operator
 * @return
 */
 public long getOptimalMemoryPerOperatorPerNode(final UserBitShared.CoreOperatorType operator) {
 return this.fragmentProfiles
 .stream()
 .flatMap(f -> f.minorFragmentProfiles
 .stream())
 .collect(Collectors.groupingBy(m -> m.endpoint.address))
 .entrySet()
 .stream()
 .collect(Collectors.toMap(Map.Entry::getKey,
 e -> e.getValue()
 .stream()
 .flatMap(m -> m.operatorProfiles
 .stream()
 .filter(o -> o.operatorId == operator.getNumber()))
 .mapToLong(o -> o.optimalMemAllocation)
 .sum()))
 .entrySet()
 .stream()
 .mapToLong(Map.Entry::getValue)
 .max()
 .orElse(0);
 }

 /**
 * Total optimal memory required (in bytes) for the query.
 * @return total optimal memory required for the query (as estimated by the RM planner).
 */
 public long getTotalOptimalMemory() {
 return getOperatorsFromProfile()
 .stream()
 .mapToLong(this::getTotalOptimalMemoryPerOperator)
 .sum();
 }

 /**
 * Returns the maximum of estimated optimal memory (in bytes) required on a drillbit.
 * @return total optimal memory required for the query (as estimated by the RM planner).
 */
 public long getTotalOptimalMemoryPerNode() {
 return getOperatorsFromProfile()
 .stream()
 .mapToLong(this::getOptimalMemoryPerOperatorPerNode)
 .sum();
 }

 /**
 * Get different operators in the profile.
 * @return a list of operators in the query profile.
Expand All@@ -233,17 +285,6 @@ public List<UserBitShared.CoreOperatorType> getOperatorsFromProfile() {
 .mapToObj(UserBitShared.CoreOperatorType::forNumber)
 .collect(Collectors.toList());
 }

 /**
 * Total optimal memory required for the query.
 * @return total optimal memory required for the query (as estimated by the RM planner).
 */
 public long getTotalOptimalMemoryEstimate() {
 return getOperatorsFromProfile()
 .stream()
 .mapToLong(this::getOptimalMemoryPerOperator)
 .sum();
 }
}



29 changes: 24 additions & 5 deletions
framework/src/main/java/org/apache/drill/test/framework/DrillRMConfig.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -36,6 +36,9 @@ public class DrillRMConfig implements DrillConfigRenderer {
 //Selector Configurations
 public static final String SELECTOR_TAG_KEY = "tag";
 public static final String SELECTOR_ACL_KEY = "acl";
 public static final String SELECTOR_OR_KEY = "or";
 public static final String SELECTOR_AND_KEY = "and";
 public static final String SELECTOR_NOT_EQUAL_KEY = "not_equal";

 //ACL Configurations
 public static final String ACL_USERS_KEY = "users";
Expand DownExpand Up@@ -74,8 +77,10 @@ public class DrillRMConfig implements DrillConfigRenderer {
 public static class SelectorConfig implements DrillConfigRenderer {

 public String tag;

 public AclConfig acl;
 public SelectorConfig not_equal;
 public List<SelectorConfig> or;
 public List<SelectorConfig> and;

 @Override
 public String render() {
Expand All@@ -98,6 +103,21 @@ public String render(final int acc) {
 sb.append(formatConfig(nextAcc, SELECTOR_ACL_KEY, acl));
 }

 if (not_equal != null) {
 ensureAtleastOneField = true;
 sb.append(formatConfig(nextAcc, SELECTOR_NOT_EQUAL_KEY, not_equal));
 }

 if (or != null) {
 ensureAtleastOneField = true;
 sb.append(formatConfig(nextAcc, SELECTOR_OR_KEY, or));
 }

 if (and != null) {
 ensureAtleastOneField = true;
 sb.append(formatConfig(nextAcc, SELECTOR_AND_KEY, and));
 }

 if(ensureAtleastOneField) {
 sb.deleteCharAt(sb.length() - 1)
 .deleteCharAt(sb.length() - 1)
Expand All@@ -119,7 +139,6 @@ public String render(final int acc) {
 public static class AclConfig implements DrillConfigRenderer {

 public List<String> users;

 public List<String> groups;

 @Override
Expand DownExpand Up@@ -163,7 +182,7 @@ public String render(final int acc) {
 public static class QueueConfig implements DrillConfigRenderer {

 @JsonProperty(QUEUE_MAX_QUERY_MEMORY_PER_NODE_KEY)
 public long maxQueryMemoryPerNodeInMB;
 public long maxQueryMemoryPerNode; //in bytes

 @JsonProperty(QUEUE_MAX_WAITING_KEY)
 public int maxWaitingQueries;
Expand All@@ -185,9 +204,9 @@ public String render(final int acc) {
 StringBuilder sb = new StringBuilder("{\n");
 final int nextAcc = acc+2;

 if (maxQueryMemoryPerNodeInMB > 0) {
 if (maxQueryMemoryPerNode > 0) {
 ensureAtleastOneField = true;
 sb.append(formatConfig(nextAcc, QUEUE_MAX_QUERY_MEMORY_PER_NODE_KEY, maxQueryMemoryPerNodeInMB));
 sb.append(formatConfig(nextAcc, QUEUE_MAX_QUERY_MEMORY_PER_NODE_KEY, maxQueryMemoryPerNode));
 }

 if (maxWaitingQueries > 0) {
Expand Down

30 changes: 24 additions & 6 deletions
framework/src/test/java/org/apache/drill/test/framework/DrillTestFrameworkUnitTests.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -21,8 +21,9 @@
import java.util.Properties;

import static org.apache.drill.test.framework.DrillTestDefaults.DRILL_EXEC_RM_CONFIG_KEY;
import static org.apache.drill.test.framework.common.DrillTestNGDefaults.BASIC_RM_CONFIG_FILEPATH;
import static org.apache.drill.test.framework.common.DrillTestNGDefaults.UNIT_GROUP;
import static org.apache.drill.test.framework.common.DrillTestNGDefaults.SAMPLE_RM_CONFIG_NAME;
import static org.apache.drill.test.framework.common.DrillTestNGDefaults.PROD_RM_CONFIG_FILEPATH;

@Test(groups = UNIT_GROUP)
public class DrillTestFrameworkUnitTests extends DrillJavaTestBase {
Expand DownExpand Up@@ -84,7 +85,7 @@ public void testQueryProfileDoesNotExist() {
 @Test(groups = UNIT_GROUP)
 public void testReadSampleRMConfigFile() {
 try {
 DrillRMConfig drillRMConfig = DrillRMConfig.load(SAMPLE_RM_CONFIG_NAME);
 DrillRMConfig drillRMConfig = DrillRMConfig.load(PROD_RM_CONFIG_FILEPATH);
 Assert.assertEquals(drillRMConfig.poolName, "root",
 "Root resource pool name did not match");

Expand All@@ -96,6 +97,22 @@ public void testReadSampleRMConfigFile() {
 }
 }

 /**
 * Test reading a sample RM config file, with complex selectors, in to a Java Bean.
 */
 @Test(groups = UNIT_GROUP)
 public void testReadComplexSelectorsRMConfigFile() {
 try {
 DrillRMConfig drillRMConfig = DrillRMConfig.load(BASIC_RM_CONFIG_FILEPATH);
 Assert.assertEquals(drillRMConfig.childPools.get(1).selector.or.size(), 2,
 "Or selector should have had 2 children!");

 } catch (Exception e) {
 e.printStackTrace();
 Assert.fail(e.getMessage());
 }
 }

 /**
 * Negative test to validate the behavior when the config file does not exist.
 */
Expand All@@ -112,7 +129,7 @@ public void testLoadConfigWhenFileDoesNotExist() throws IOException {
 @Test(groups = UNIT_GROUP)
 public void testConfigFileRenderer() {
 try {
 DrillRMConfig drillRMConfig = DrillRMConfig.load(SAMPLE_RM_CONFIG_NAME);
 DrillRMConfig drillRMConfig = DrillRMConfig.load(PROD_RM_CONFIG_FILEPATH);
 Assert.assertEquals(drillRMConfig.poolName, "root",
 "Root resource pool name did not match");

Expand DownExpand Up@@ -149,7 +166,7 @@ public void testWriteRMConfigToFile() throws IOException {
 }

 try (BufferedWriter writer = new BufferedWriter(new FileWriter(filePath))) {
 DrillRMConfig drillRMConfig = DrillRMConfig.load(SAMPLE_RM_CONFIG_NAME);
 DrillRMConfig drillRMConfig = DrillRMConfig.load(PROD_RM_CONFIG_FILEPATH);
 writer.write(DRILL_EXEC_RM_CONFIG_KEY + ":" + drillRMConfig.render());
 }

Expand DownExpand Up@@ -178,14 +195,15 @@ public void testTotalMemoryForQueryProfile() {
 DrillQueryProfile profile = Utils.getQueryProfile(queryId);
 Assert.assertEquals(profile.queryId, queryId);

 long rmMemEstimate = profile.getTotalOptimalMemoryEstimate();
 long rmMemEstimate = profile.getTotalOptimalMemoryPerNode();
 LOG.info("Memory estimated by RM planner: " + rmMemEstimate);
 Assert.assertTrue(rmMemEstimate > 0,
 "RM estimated memory should be greater than 0");
 List<UserBitShared.CoreOperatorType> operators = profile.getOperatorsFromProfile();
 Assert.assertTrue(operators.size() > 0,
 "Number of operators in the profile should be greater than 0");
 operators.forEach(LOG::info);
 operators.forEach(o -> LOG.info("Operator: " + o + ", Optimal Memory per Node in bytes: " +
 profile.getOptimalMemoryPerOperatorPerNode(o)));
 } catch (Exception e) {
 e.printStackTrace();
 Assert.fail(e.getMessage());
Expand Down

11 changes: 7 additions & 4 deletions
framework/src/test/java/org/apache/drill/test/framework/common/DrillTestNGDefaults.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -5,13 +5,16 @@
public final class DrillTestNGDefaults {
 public static final String FUNCTIONAL_GROUP = "functional";
 public static final String UNIT_GROUP = "unit";
 public static final String SAMPLE_RM_CONFIG_NAME =
 DrillTestDefaults.CWD + "/src/test/resources/sample-drill-rm-override.conf";
 public static final String BASIC_RM_CONFIG_NAME =
 DrillTestDefaults.CWD + "/src/test/resources/basic-drill-rm-override.conf";
 public static final String PROD_RM_CONFIG_FILEPATH =
 DrillTestDefaults.CWD + "/src/test/resources/prod-rm.conf";
 public static final String BASIC_RM_CONFIG_FILEPATH =
 DrillTestDefaults.CWD + "/src/test/resources/basic-rm.conf";
 public static final String DEFAULT_RM_CONFIG_FILEPATH =
 DrillTestDefaults.CWD + "/src/test/resources/default-rm.conf";

 public static String CONNECTION_URL_FOR_DRILLBIT(final String hostnameOrIp) {
 return String.format("jdbc:drill:drillbit=%s", hostnameOrIp);
 }
 public static final String NO_RESOURCE_POOL_ERROR = "No resource pools to choose from for the query";
 public static final String TPCH_01_PARQUET_SCHEMA = "dfs.drilltestdirtpch01parquet";
}
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
// Strip utm_, fbclid, gclid, etc. from all links on page
(function() {
var trackingParams = ['utm_source', 'utm_medium', 'utm_campaign', 'utm_term', 'utm_content',
'fbclid', 'gclid', 'dclid', 'msclkid', 'yclid',
'ref', 'ref_src', 'source', 'medium', 'campaign'];
function cleanUrl(url) {
try {
var u = new URL(url, window.location.origin);
var changed = false;
trackingParams.forEach(function(p) {
if (u.searchParams.has(p)) {
u.searchParams.delete(p);
changed = true;
}
});
return changed ? u.toString() : url;
} catch (e) {
return url;
}
}
function cleanLinks() {
document.querySelectorAll('a[href]').forEach(function(a) {
var clean = cleanUrl(a.href);
if (clean !== a.href) a.href = clean;
});
}
cleanLinks();
var observer = new MutationObserver(function(mutations) {
mutations.forEach(function(m) {
m.addedNodes.forEach(function(node) {
if (node.nodeType === 1) {
if (node.tagName === 'A') cleanLinks();
node.querySelectorAll('a[href]').forEach(function(a) {
var clean = cleanUrl(a.href);
if (clean !== a.href) a.href = clean;
});
}
});
});
});
observer.observe(document.body, { childList: true, subtree: true });
})();
}
} catch(__e) { console.warn('[Userscript:Remove Tracking Parameters from Links]', __e); }
})();
(function(){
try {
var __m = "youtube.com";
var __re = new RegExp('^' + "youtube\\.com" + '

Skip to content
RM planner test and framework changes#581

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Open

abhidotravi
wants to merge
7
commits into

base:master
Choose a base branch

from

 Open


RM planner test and framework changes
#581
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


7 commits


Select commit
Hold shift + click to select a range
a38429f

RM planner test and framework changes


abhidotravi Apr 24, 2019
8f11f93

Add support for complex selector


abhidotravi Apr 24, 2019
6b07e4a

Add a run script to run tests


abhidotravi Apr 25, 2019
15a5fbd

Add help to run script


abhidotravi Apr 25, 2019
0d509c9

Minor cosmetic change


abhidotravi Apr 25, 2019
483ee23

More minor cosmetic changes


abhidotravi Apr 25, 2019
a566d6a

Add more tests for Planner and Selector


abhidotravi May 2, 2019

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


13 changes: 13 additions & 0 deletions
bin/run_testng

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
#export MAVEN_OPTS=$MAVEN_OPTS" -agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=5006"
bin=`dirname "${BASH_SOURCE-$0}"`
bin=`cd "$bin">/dev/null; pwd`
cd ${bin}/../framework
source ../conf/drillTestConfig.properties
if [[ $1 == "-Dtest="* ]]; then
 mvn test $*
elif [[ $1 == "-h" || $1 == "-help" ]]; then
 echo "bin/runtestng [-Dtest=<class_name>[#<method_name>]]"
else
 echo "Running the entire Suite"
 mvn test -Ptestng $*
fi

69 changes: 55 additions & 14 deletions
framework/src/main/java/org/apache/drill/test/framework/DrillQueryProfile.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -6,6 +6,7 @@
import oadd.org.apache.drill.exec.proto.UserBitShared;

import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;

@JsonIgnoreProperties(ignoreUnknown = true)
Expand DownExpand Up@@ -203,12 +204,12 @@ public String toString() {
 }

 /**
 * Get optimal memory allocated per operator.
 * Utility parses the DrillQueryProfile
 * Get total optimal memory allocated (in bytes) for specified operator, across all drillbits.
 *
 * @param operator
 * @return
 */
 public long getOptimalMemoryPerOperator(final UserBitShared.CoreOperatorType operator) {
 public long getTotalOptimalMemoryPerOperator(final UserBitShared.CoreOperatorType operator) {
 return this.fragmentProfiles
 .stream()
 .flatMap(f -> f.minorFragmentProfiles
Expand All@@ -219,6 +220,57 @@ public long getOptimalMemoryPerOperator(final UserBitShared.CoreOperatorType ope
 .sum();
 }

 /**
 * Returns the max of optimal memory allocated (in bytes) to specified operator on a drillbit.
 *
 * @param operator
 * @return
 */
 public long getOptimalMemoryPerOperatorPerNode(final UserBitShared.CoreOperatorType operator) {
 return this.fragmentProfiles
 .stream()
 .flatMap(f -> f.minorFragmentProfiles
 .stream())
 .collect(Collectors.groupingBy(m -> m.endpoint.address))
 .entrySet()
 .stream()
 .collect(Collectors.toMap(Map.Entry::getKey,
 e -> e.getValue()
 .stream()
 .flatMap(m -> m.operatorProfiles
 .stream()
 .filter(o -> o.operatorId == operator.getNumber()))
 .mapToLong(o -> o.optimalMemAllocation)
 .sum()))
 .entrySet()
 .stream()
 .mapToLong(Map.Entry::getValue)
 .max()
 .orElse(0);
 }

 /**
 * Total optimal memory required (in bytes) for the query.
 * @return total optimal memory required for the query (as estimated by the RM planner).
 */
 public long getTotalOptimalMemory() {
 return getOperatorsFromProfile()
 .stream()
 .mapToLong(this::getTotalOptimalMemoryPerOperator)
 .sum();
 }

 /**
 * Returns the maximum of estimated optimal memory (in bytes) required on a drillbit.
 * @return total optimal memory required for the query (as estimated by the RM planner).
 */
 public long getTotalOptimalMemoryPerNode() {
 return getOperatorsFromProfile()
 .stream()
 .mapToLong(this::getOptimalMemoryPerOperatorPerNode)
 .sum();
 }

 /**
 * Get different operators in the profile.
 * @return a list of operators in the query profile.
Expand All@@ -233,17 +285,6 @@ public List<UserBitShared.CoreOperatorType> getOperatorsFromProfile() {
 .mapToObj(UserBitShared.CoreOperatorType::forNumber)
 .collect(Collectors.toList());
 }

 /**
 * Total optimal memory required for the query.
 * @return total optimal memory required for the query (as estimated by the RM planner).
 */
 public long getTotalOptimalMemoryEstimate() {
 return getOperatorsFromProfile()
 .stream()
 .mapToLong(this::getOptimalMemoryPerOperator)
 .sum();
 }
}



29 changes: 24 additions & 5 deletions
framework/src/main/java/org/apache/drill/test/framework/DrillRMConfig.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -36,6 +36,9 @@ public class DrillRMConfig implements DrillConfigRenderer {
 //Selector Configurations
 public static final String SELECTOR_TAG_KEY = "tag";
 public static final String SELECTOR_ACL_KEY = "acl";
 public static final String SELECTOR_OR_KEY = "or";
 public static final String SELECTOR_AND_KEY = "and";
 public static final String SELECTOR_NOT_EQUAL_KEY = "not_equal";

 //ACL Configurations
 public static final String ACL_USERS_KEY = "users";
Expand DownExpand Up@@ -74,8 +77,10 @@ public class DrillRMConfig implements DrillConfigRenderer {
 public static class SelectorConfig implements DrillConfigRenderer {

 public String tag;

 public AclConfig acl;
 public SelectorConfig not_equal;
 public List<SelectorConfig> or;
 public List<SelectorConfig> and;

 @Override
 public String render() {
Expand All@@ -98,6 +103,21 @@ public String render(final int acc) {
 sb.append(formatConfig(nextAcc, SELECTOR_ACL_KEY, acl));
 }

 if (not_equal != null) {
 ensureAtleastOneField = true;
 sb.append(formatConfig(nextAcc, SELECTOR_NOT_EQUAL_KEY, not_equal));
 }

 if (or != null) {
 ensureAtleastOneField = true;
 sb.append(formatConfig(nextAcc, SELECTOR_OR_KEY, or));
 }

 if (and != null) {
 ensureAtleastOneField = true;
 sb.append(formatConfig(nextAcc, SELECTOR_AND_KEY, and));
 }

 if(ensureAtleastOneField) {
 sb.deleteCharAt(sb.length() - 1)
 .deleteCharAt(sb.length() - 1)
Expand All@@ -119,7 +139,6 @@ public String render(final int acc) {
 public static class AclConfig implements DrillConfigRenderer {

 public List<String> users;

 public List<String> groups;

 @Override
Expand DownExpand Up@@ -163,7 +182,7 @@ public String render(final int acc) {
 public static class QueueConfig implements DrillConfigRenderer {

 @JsonProperty(QUEUE_MAX_QUERY_MEMORY_PER_NODE_KEY)
 public long maxQueryMemoryPerNodeInMB;
 public long maxQueryMemoryPerNode; //in bytes

 @JsonProperty(QUEUE_MAX_WAITING_KEY)
 public int maxWaitingQueries;
Expand All@@ -185,9 +204,9 @@ public String render(final int acc) {
 StringBuilder sb = new StringBuilder("{\n");
 final int nextAcc = acc+2;

 if (maxQueryMemoryPerNodeInMB > 0) {
 if (maxQueryMemoryPerNode > 0) {
 ensureAtleastOneField = true;
 sb.append(formatConfig(nextAcc, QUEUE_MAX_QUERY_MEMORY_PER_NODE_KEY, maxQueryMemoryPerNodeInMB));
 sb.append(formatConfig(nextAcc, QUEUE_MAX_QUERY_MEMORY_PER_NODE_KEY, maxQueryMemoryPerNode));
 }

 if (maxWaitingQueries > 0) {
Expand Down

30 changes: 24 additions & 6 deletions
framework/src/test/java/org/apache/drill/test/framework/DrillTestFrameworkUnitTests.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -21,8 +21,9 @@
import java.util.Properties;

import static org.apache.drill.test.framework.DrillTestDefaults.DRILL_EXEC_RM_CONFIG_KEY;
import static org.apache.drill.test.framework.common.DrillTestNGDefaults.BASIC_RM_CONFIG_FILEPATH;
import static org.apache.drill.test.framework.common.DrillTestNGDefaults.UNIT_GROUP;
import static org.apache.drill.test.framework.common.DrillTestNGDefaults.SAMPLE_RM_CONFIG_NAME;
import static org.apache.drill.test.framework.common.DrillTestNGDefaults.PROD_RM_CONFIG_FILEPATH;

@Test(groups = UNIT_GROUP)
public class DrillTestFrameworkUnitTests extends DrillJavaTestBase {
Expand DownExpand Up@@ -84,7 +85,7 @@ public void testQueryProfileDoesNotExist() {
 @Test(groups = UNIT_GROUP)
 public void testReadSampleRMConfigFile() {
 try {
 DrillRMConfig drillRMConfig = DrillRMConfig.load(SAMPLE_RM_CONFIG_NAME);
 DrillRMConfig drillRMConfig = DrillRMConfig.load(PROD_RM_CONFIG_FILEPATH);
 Assert.assertEquals(drillRMConfig.poolName, "root",
 "Root resource pool name did not match");

Expand All@@ -96,6 +97,22 @@ public void testReadSampleRMConfigFile() {
 }
 }

 /**
 * Test reading a sample RM config file, with complex selectors, in to a Java Bean.
 */
 @Test(groups = UNIT_GROUP)
 public void testReadComplexSelectorsRMConfigFile() {
 try {
 DrillRMConfig drillRMConfig = DrillRMConfig.load(BASIC_RM_CONFIG_FILEPATH);
 Assert.assertEquals(drillRMConfig.childPools.get(1).selector.or.size(), 2,
 "Or selector should have had 2 children!");

 } catch (Exception e) {
 e.printStackTrace();
 Assert.fail(e.getMessage());
 }
 }

 /**
 * Negative test to validate the behavior when the config file does not exist.
 */
Expand All@@ -112,7 +129,7 @@ public void testLoadConfigWhenFileDoesNotExist() throws IOException {
 @Test(groups = UNIT_GROUP)
 public void testConfigFileRenderer() {
 try {
 DrillRMConfig drillRMConfig = DrillRMConfig.load(SAMPLE_RM_CONFIG_NAME);
 DrillRMConfig drillRMConfig = DrillRMConfig.load(PROD_RM_CONFIG_FILEPATH);
 Assert.assertEquals(drillRMConfig.poolName, "root",
 "Root resource pool name did not match");

Expand DownExpand Up@@ -149,7 +166,7 @@ public void testWriteRMConfigToFile() throws IOException {
 }

 try (BufferedWriter writer = new BufferedWriter(new FileWriter(filePath))) {
 DrillRMConfig drillRMConfig = DrillRMConfig.load(SAMPLE_RM_CONFIG_NAME);
 DrillRMConfig drillRMConfig = DrillRMConfig.load(PROD_RM_CONFIG_FILEPATH);
 writer.write(DRILL_EXEC_RM_CONFIG_KEY + ":" + drillRMConfig.render());
 }

Expand DownExpand Up@@ -178,14 +195,15 @@ public void testTotalMemoryForQueryProfile() {
 DrillQueryProfile profile = Utils.getQueryProfile(queryId);
 Assert.assertEquals(profile.queryId, queryId);

 long rmMemEstimate = profile.getTotalOptimalMemoryEstimate();
 long rmMemEstimate = profile.getTotalOptimalMemoryPerNode();
 LOG.info("Memory estimated by RM planner: " + rmMemEstimate);
 Assert.assertTrue(rmMemEstimate > 0,
 "RM estimated memory should be greater than 0");
 List<UserBitShared.CoreOperatorType> operators = profile.getOperatorsFromProfile();
 Assert.assertTrue(operators.size() > 0,
 "Number of operators in the profile should be greater than 0");
 operators.forEach(LOG::info);
 operators.forEach(o -> LOG.info("Operator: " + o + ", Optimal Memory per Node in bytes: " +
 profile.getOptimalMemoryPerOperatorPerNode(o)));
 } catch (Exception e) {
 e.printStackTrace();
 Assert.fail(e.getMessage());
Expand Down

11 changes: 7 additions & 4 deletions
framework/src/test/java/org/apache/drill/test/framework/common/DrillTestNGDefaults.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -5,13 +5,16 @@
public final class DrillTestNGDefaults {
 public static final String FUNCTIONAL_GROUP = "functional";
 public static final String UNIT_GROUP = "unit";
 public static final String SAMPLE_RM_CONFIG_NAME =
 DrillTestDefaults.CWD + "/src/test/resources/sample-drill-rm-override.conf";
 public static final String BASIC_RM_CONFIG_NAME =
 DrillTestDefaults.CWD + "/src/test/resources/basic-drill-rm-override.conf";
 public static final String PROD_RM_CONFIG_FILEPATH =
 DrillTestDefaults.CWD + "/src/test/resources/prod-rm.conf";
 public static final String BASIC_RM_CONFIG_FILEPATH =
 DrillTestDefaults.CWD + "/src/test/resources/basic-rm.conf";
 public static final String DEFAULT_RM_CONFIG_FILEPATH =
 DrillTestDefaults.CWD + "/src/test/resources/default-rm.conf";

 public static String CONNECTION_URL_FOR_DRILLBIT(final String hostnameOrIp) {
 return String.format("jdbc:drill:drillbit=%s", hostnameOrIp);
 }
 public static final String NO_RESOURCE_POOL_ERROR = "No resource pools to choose from for the query";
 public static final String TPCH_01_PARQUET_SCHEMA = "dfs.drilltestdirtpch01parquet";
}
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
// Auto-enable theater mode on YouTube
(function() {
function tryTheater() {
var btn = document.querySelector('button[aria-label="Theater mode"], ytd-player #player button[title="Theater mode"]');
if (btn && !btn.classList.contains('activated')) {
btn.click();
}
}
// Try immediately
tryTheater();
// Try after navigation (SPA)
var lastUrl = location.href;
setInterval(function() {
if (location.href !== lastUrl) {
lastUrl = location.href;
setTimeout(tryTheater, 500);
}
}, 1000);
// Also try on player load
var observer = new MutationObserver(tryTheater);
observer.observe(document.body, { childList: true, subtree: true });
})();
}
} catch(__e) { console.warn('[Userscript:YouTube Theater Mode Default]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
RM planner test and framework changes#581

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Open

abhidotravi
wants to merge
7
commits into

base:master
Choose a base branch

from

 Open


RM planner test and framework changes
#581
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


7 commits


Select commit
Hold shift + click to select a range
a38429f

RM planner test and framework changes


abhidotravi Apr 24, 2019
8f11f93

Add support for complex selector


abhidotravi Apr 24, 2019
6b07e4a

Add a run script to run tests


abhidotravi Apr 25, 2019
15a5fbd

Add help to run script


abhidotravi Apr 25, 2019
0d509c9

Minor cosmetic change


abhidotravi Apr 25, 2019
483ee23

More minor cosmetic changes


abhidotravi Apr 25, 2019
a566d6a

Add more tests for Planner and Selector


abhidotravi May 2, 2019

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


13 changes: 13 additions & 0 deletions
bin/run_testng

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
#export MAVEN_OPTS=$MAVEN_OPTS" -agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=5006"
bin=`dirname "${BASH_SOURCE-$0}"`
bin=`cd "$bin">/dev/null; pwd`
cd ${bin}/../framework
source ../conf/drillTestConfig.properties
if [[ $1 == "-Dtest="* ]]; then
 mvn test $*
elif [[ $1 == "-h" || $1 == "-help" ]]; then
 echo "bin/runtestng [-Dtest=<class_name>[#<method_name>]]"
else
 echo "Running the entire Suite"
 mvn test -Ptestng $*
fi

69 changes: 55 additions & 14 deletions
framework/src/main/java/org/apache/drill/test/framework/DrillQueryProfile.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -6,6 +6,7 @@
import oadd.org.apache.drill.exec.proto.UserBitShared;

import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;

@JsonIgnoreProperties(ignoreUnknown = true)
Expand DownExpand Up@@ -203,12 +204,12 @@ public String toString() {
 }

 /**
 * Get optimal memory allocated per operator.
 * Utility parses the DrillQueryProfile
 * Get total optimal memory allocated (in bytes) for specified operator, across all drillbits.
 *
 * @param operator
 * @return
 */
 public long getOptimalMemoryPerOperator(final UserBitShared.CoreOperatorType operator) {
 public long getTotalOptimalMemoryPerOperator(final UserBitShared.CoreOperatorType operator) {
 return this.fragmentProfiles
 .stream()
 .flatMap(f -> f.minorFragmentProfiles
Expand All@@ -219,6 +220,57 @@ public long getOptimalMemoryPerOperator(final UserBitShared.CoreOperatorType ope
 .sum();
 }

 /**
 * Returns the max of optimal memory allocated (in bytes) to specified operator on a drillbit.
 *
 * @param operator
 * @return
 */
 public long getOptimalMemoryPerOperatorPerNode(final UserBitShared.CoreOperatorType operator) {
 return this.fragmentProfiles
 .stream()
 .flatMap(f -> f.minorFragmentProfiles
 .stream())
 .collect(Collectors.groupingBy(m -> m.endpoint.address))
 .entrySet()
 .stream()
 .collect(Collectors.toMap(Map.Entry::getKey,
 e -> e.getValue()
 .stream()
 .flatMap(m -> m.operatorProfiles
 .stream()
 .filter(o -> o.operatorId == operator.getNumber()))
 .mapToLong(o -> o.optimalMemAllocation)
 .sum()))
 .entrySet()
 .stream()
 .mapToLong(Map.Entry::getValue)
 .max()
 .orElse(0);
 }

 /**
 * Total optimal memory required (in bytes) for the query.
 * @return total optimal memory required for the query (as estimated by the RM planner).
 */
 public long getTotalOptimalMemory() {
 return getOperatorsFromProfile()
 .stream()
 .mapToLong(this::getTotalOptimalMemoryPerOperator)
 .sum();
 }

 /**
 * Returns the maximum of estimated optimal memory (in bytes) required on a drillbit.
 * @return total optimal memory required for the query (as estimated by the RM planner).
 */
 public long getTotalOptimalMemoryPerNode() {
 return getOperatorsFromProfile()
 .stream()
 .mapToLong(this::getOptimalMemoryPerOperatorPerNode)
 .sum();
 }

 /**
 * Get different operators in the profile.
 * @return a list of operators in the query profile.
Expand All@@ -233,17 +285,6 @@ public List<UserBitShared.CoreOperatorType> getOperatorsFromProfile() {
 .mapToObj(UserBitShared.CoreOperatorType::forNumber)
 .collect(Collectors.toList());
 }

 /**
 * Total optimal memory required for the query.
 * @return total optimal memory required for the query (as estimated by the RM planner).
 */
 public long getTotalOptimalMemoryEstimate() {
 return getOperatorsFromProfile()
 .stream()
 .mapToLong(this::getOptimalMemoryPerOperator)
 .sum();
 }
}



29 changes: 24 additions & 5 deletions
framework/src/main/java/org/apache/drill/test/framework/DrillRMConfig.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -36,6 +36,9 @@ public class DrillRMConfig implements DrillConfigRenderer {
 //Selector Configurations
 public static final String SELECTOR_TAG_KEY = "tag";
 public static final String SELECTOR_ACL_KEY = "acl";
 public static final String SELECTOR_OR_KEY = "or";
 public static final String SELECTOR_AND_KEY = "and";
 public static final String SELECTOR_NOT_EQUAL_KEY = "not_equal";

 //ACL Configurations
 public static final String ACL_USERS_KEY = "users";
Expand DownExpand Up@@ -74,8 +77,10 @@ public class DrillRMConfig implements DrillConfigRenderer {
 public static class SelectorConfig implements DrillConfigRenderer {

 public String tag;

 public AclConfig acl;
 public SelectorConfig not_equal;
 public List<SelectorConfig> or;
 public List<SelectorConfig> and;

 @Override
 public String render() {
Expand All@@ -98,6 +103,21 @@ public String render(final int acc) {
 sb.append(formatConfig(nextAcc, SELECTOR_ACL_KEY, acl));
 }

 if (not_equal != null) {
 ensureAtleastOneField = true;
 sb.append(formatConfig(nextAcc, SELECTOR_NOT_EQUAL_KEY, not_equal));
 }

 if (or != null) {
 ensureAtleastOneField = true;
 sb.append(formatConfig(nextAcc, SELECTOR_OR_KEY, or));
 }

 if (and != null) {
 ensureAtleastOneField = true;
 sb.append(formatConfig(nextAcc, SELECTOR_AND_KEY, and));
 }

 if(ensureAtleastOneField) {
 sb.deleteCharAt(sb.length() - 1)
 .deleteCharAt(sb.length() - 1)
Expand All@@ -119,7 +139,6 @@ public String render(final int acc) {
 public static class AclConfig implements DrillConfigRenderer {

 public List<String> users;

 public List<String> groups;

 @Override
Expand DownExpand Up@@ -163,7 +182,7 @@ public String render(final int acc) {
 public static class QueueConfig implements DrillConfigRenderer {

 @JsonProperty(QUEUE_MAX_QUERY_MEMORY_PER_NODE_KEY)
 public long maxQueryMemoryPerNodeInMB;
 public long maxQueryMemoryPerNode; //in bytes

 @JsonProperty(QUEUE_MAX_WAITING_KEY)
 public int maxWaitingQueries;
Expand All@@ -185,9 +204,9 @@ public String render(final int acc) {
 StringBuilder sb = new StringBuilder("{\n");
 final int nextAcc = acc+2;

 if (maxQueryMemoryPerNodeInMB > 0) {
 if (maxQueryMemoryPerNode > 0) {
 ensureAtleastOneField = true;
 sb.append(formatConfig(nextAcc, QUEUE_MAX_QUERY_MEMORY_PER_NODE_KEY, maxQueryMemoryPerNodeInMB));
 sb.append(formatConfig(nextAcc, QUEUE_MAX_QUERY_MEMORY_PER_NODE_KEY, maxQueryMemoryPerNode));
 }

 if (maxWaitingQueries > 0) {
Expand Down

30 changes: 24 additions & 6 deletions
framework/src/test/java/org/apache/drill/test/framework/DrillTestFrameworkUnitTests.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -21,8 +21,9 @@
import java.util.Properties;

import static org.apache.drill.test.framework.DrillTestDefaults.DRILL_EXEC_RM_CONFIG_KEY;
import static org.apache.drill.test.framework.common.DrillTestNGDefaults.BASIC_RM_CONFIG_FILEPATH;
import static org.apache.drill.test.framework.common.DrillTestNGDefaults.UNIT_GROUP;
import static org.apache.drill.test.framework.common.DrillTestNGDefaults.SAMPLE_RM_CONFIG_NAME;
import static org.apache.drill.test.framework.common.DrillTestNGDefaults.PROD_RM_CONFIG_FILEPATH;

@Test(groups = UNIT_GROUP)
public class DrillTestFrameworkUnitTests extends DrillJavaTestBase {
Expand DownExpand Up@@ -84,7 +85,7 @@ public void testQueryProfileDoesNotExist() {
 @Test(groups = UNIT_GROUP)
 public void testReadSampleRMConfigFile() {
 try {
 DrillRMConfig drillRMConfig = DrillRMConfig.load(SAMPLE_RM_CONFIG_NAME);
 DrillRMConfig drillRMConfig = DrillRMConfig.load(PROD_RM_CONFIG_FILEPATH);
 Assert.assertEquals(drillRMConfig.poolName, "root",
 "Root resource pool name did not match");

Expand All@@ -96,6 +97,22 @@ public void testReadSampleRMConfigFile() {
 }
 }

 /**
 * Test reading a sample RM config file, with complex selectors, in to a Java Bean.
 */
 @Test(groups = UNIT_GROUP)
 public void testReadComplexSelectorsRMConfigFile() {
 try {
 DrillRMConfig drillRMConfig = DrillRMConfig.load(BASIC_RM_CONFIG_FILEPATH);
 Assert.assertEquals(drillRMConfig.childPools.get(1).selector.or.size(), 2,
 "Or selector should have had 2 children!");

 } catch (Exception e) {
 e.printStackTrace();
 Assert.fail(e.getMessage());
 }
 }

 /**
 * Negative test to validate the behavior when the config file does not exist.
 */
Expand All@@ -112,7 +129,7 @@ public void testLoadConfigWhenFileDoesNotExist() throws IOException {
 @Test(groups = UNIT_GROUP)
 public void testConfigFileRenderer() {
 try {
 DrillRMConfig drillRMConfig = DrillRMConfig.load(SAMPLE_RM_CONFIG_NAME);
 DrillRMConfig drillRMConfig = DrillRMConfig.load(PROD_RM_CONFIG_FILEPATH);
 Assert.assertEquals(drillRMConfig.poolName, "root",
 "Root resource pool name did not match");

Expand DownExpand Up@@ -149,7 +166,7 @@ public void testWriteRMConfigToFile() throws IOException {
 }

 try (BufferedWriter writer = new BufferedWriter(new FileWriter(filePath))) {
 DrillRMConfig drillRMConfig = DrillRMConfig.load(SAMPLE_RM_CONFIG_NAME);
 DrillRMConfig drillRMConfig = DrillRMConfig.load(PROD_RM_CONFIG_FILEPATH);
 writer.write(DRILL_EXEC_RM_CONFIG_KEY + ":" + drillRMConfig.render());
 }

Expand DownExpand Up@@ -178,14 +195,15 @@ public void testTotalMemoryForQueryProfile() {
 DrillQueryProfile profile = Utils.getQueryProfile(queryId);
 Assert.assertEquals(profile.queryId, queryId);

 long rmMemEstimate = profile.getTotalOptimalMemoryEstimate();
 long rmMemEstimate = profile.getTotalOptimalMemoryPerNode();
 LOG.info("Memory estimated by RM planner: " + rmMemEstimate);
 Assert.assertTrue(rmMemEstimate > 0,
 "RM estimated memory should be greater than 0");
 List<UserBitShared.CoreOperatorType> operators = profile.getOperatorsFromProfile();
 Assert.assertTrue(operators.size() > 0,
 "Number of operators in the profile should be greater than 0");
 operators.forEach(LOG::info);
 operators.forEach(o -> LOG.info("Operator: " + o + ", Optimal Memory per Node in bytes: " +
 profile.getOptimalMemoryPerOperatorPerNode(o)));
 } catch (Exception e) {
 e.printStackTrace();
 Assert.fail(e.getMessage());
Expand Down

11 changes: 7 additions & 4 deletions
framework/src/test/java/org/apache/drill/test/framework/common/DrillTestNGDefaults.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -5,13 +5,16 @@
public final class DrillTestNGDefaults {
 public static final String FUNCTIONAL_GROUP = "functional";
 public static final String UNIT_GROUP = "unit";
 public static final String SAMPLE_RM_CONFIG_NAME =
 DrillTestDefaults.CWD + "/src/test/resources/sample-drill-rm-override.conf";
 public static final String BASIC_RM_CONFIG_NAME =
 DrillTestDefaults.CWD + "/src/test/resources/basic-drill-rm-override.conf";
 public static final String PROD_RM_CONFIG_FILEPATH =
 DrillTestDefaults.CWD + "/src/test/resources/prod-rm.conf";
 public static final String BASIC_RM_CONFIG_FILEPATH =
 DrillTestDefaults.CWD + "/src/test/resources/basic-rm.conf";
 public static final String DEFAULT_RM_CONFIG_FILEPATH =
 DrillTestDefaults.CWD + "/src/test/resources/default-rm.conf";

 public static String CONNECTION_URL_FOR_DRILLBIT(final String hostnameOrIp) {
 return String.format("jdbc:drill:drillbit=%s", hostnameOrIp);
 }
 public static final String NO_RESOURCE_POOL_ERROR = "No resource pools to choose from for the query";
 public static final String TPCH_01_PARQUET_SCHEMA = "dfs.drilltestdirtpch01parquet";
}
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
// Remove or un-stick sticky/fixed headers that block content
(function() {
function unstick() {
document.querySelectorAll('header, nav, [role="banner"], .header, .navbar, .sticky, .fixed-top, [style*="position: fixed"], [style*="position:sticky"]').forEach(function(el) {
if (el.style.position === 'fixed' || el.style.position === 'sticky' || getComputedStyle(el).position === 'fixed' || getComputedStyle(el).position === 'sticky') {
el.style.position = 'static';
el.style.top = 'auto';
el.style.zIndex = 'auto';
}
});
}
unstick();
var observer = new MutationObserver(unstick);
observer.observe(document.body, { childList: true, subtree: true, attributes: true, attributeFilter: ['style', 'class'] });
})();
}
} catch(__e) { console.warn('[Userscript:Kill Sticky Headers]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
RM planner test and framework changes#581

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Open

abhidotravi
wants to merge
7
commits into

base:master
Choose a base branch

from

 Open


RM planner test and framework changes
#581
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


7 commits


Select commit
Hold shift + click to select a range
a38429f

RM planner test and framework changes


abhidotravi Apr 24, 2019
8f11f93

Add support for complex selector


abhidotravi Apr 24, 2019
6b07e4a

Add a run script to run tests


abhidotravi Apr 25, 2019
15a5fbd

Add help to run script


abhidotravi Apr 25, 2019
0d509c9

Minor cosmetic change


abhidotravi Apr 25, 2019
483ee23

More minor cosmetic changes


abhidotravi Apr 25, 2019
a566d6a

Add more tests for Planner and Selector


abhidotravi May 2, 2019

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


13 changes: 13 additions & 0 deletions
bin/run_testng

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
#export MAVEN_OPTS=$MAVEN_OPTS" -agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=5006"
bin=`dirname "${BASH_SOURCE-$0}"`
bin=`cd "$bin">/dev/null; pwd`
cd ${bin}/../framework
source ../conf/drillTestConfig.properties
if [[ $1 == "-Dtest="* ]]; then
 mvn test $*
elif [[ $1 == "-h" || $1 == "-help" ]]; then
 echo "bin/runtestng [-Dtest=<class_name>[#<method_name>]]"
else
 echo "Running the entire Suite"
 mvn test -Ptestng $*
fi

69 changes: 55 additions & 14 deletions
framework/src/main/java/org/apache/drill/test/framework/DrillQueryProfile.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -6,6 +6,7 @@
import oadd.org.apache.drill.exec.proto.UserBitShared;

import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;

@JsonIgnoreProperties(ignoreUnknown = true)
Expand DownExpand Up@@ -203,12 +204,12 @@ public String toString() {
 }

 /**
 * Get optimal memory allocated per operator.
 * Utility parses the DrillQueryProfile
 * Get total optimal memory allocated (in bytes) for specified operator, across all drillbits.
 *
 * @param operator
 * @return
 */
 public long getOptimalMemoryPerOperator(final UserBitShared.CoreOperatorType operator) {
 public long getTotalOptimalMemoryPerOperator(final UserBitShared.CoreOperatorType operator) {
 return this.fragmentProfiles
 .stream()
 .flatMap(f -> f.minorFragmentProfiles
Expand All@@ -219,6 +220,57 @@ public long getOptimalMemoryPerOperator(final UserBitShared.CoreOperatorType ope
 .sum();
 }

 /**
 * Returns the max of optimal memory allocated (in bytes) to specified operator on a drillbit.
 *
 * @param operator
 * @return
 */
 public long getOptimalMemoryPerOperatorPerNode(final UserBitShared.CoreOperatorType operator) {
 return this.fragmentProfiles
 .stream()
 .flatMap(f -> f.minorFragmentProfiles
 .stream())
 .collect(Collectors.groupingBy(m -> m.endpoint.address))
 .entrySet()
 .stream()
 .collect(Collectors.toMap(Map.Entry::getKey,
 e -> e.getValue()
 .stream()
 .flatMap(m -> m.operatorProfiles
 .stream()
 .filter(o -> o.operatorId == operator.getNumber()))
 .mapToLong(o -> o.optimalMemAllocation)
 .sum()))
 .entrySet()
 .stream()
 .mapToLong(Map.Entry::getValue)
 .max()
 .orElse(0);
 }

 /**
 * Total optimal memory required (in bytes) for the query.
 * @return total optimal memory required for the query (as estimated by the RM planner).
 */
 public long getTotalOptimalMemory() {
 return getOperatorsFromProfile()
 .stream()
 .mapToLong(this::getTotalOptimalMemoryPerOperator)
 .sum();
 }

 /**
 * Returns the maximum of estimated optimal memory (in bytes) required on a drillbit.
 * @return total optimal memory required for the query (as estimated by the RM planner).
 */
 public long getTotalOptimalMemoryPerNode() {
 return getOperatorsFromProfile()
 .stream()
 .mapToLong(this::getOptimalMemoryPerOperatorPerNode)
 .sum();
 }

 /**
 * Get different operators in the profile.
 * @return a list of operators in the query profile.
Expand All@@ -233,17 +285,6 @@ public List<UserBitShared.CoreOperatorType> getOperatorsFromProfile() {
 .mapToObj(UserBitShared.CoreOperatorType::forNumber)
 .collect(Collectors.toList());
 }

 /**
 * Total optimal memory required for the query.
 * @return total optimal memory required for the query (as estimated by the RM planner).
 */
 public long getTotalOptimalMemoryEstimate() {
 return getOperatorsFromProfile()
 .stream()
 .mapToLong(this::getOptimalMemoryPerOperator)
 .sum();
 }
}



29 changes: 24 additions & 5 deletions
framework/src/main/java/org/apache/drill/test/framework/DrillRMConfig.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -36,6 +36,9 @@ public class DrillRMConfig implements DrillConfigRenderer {
 //Selector Configurations
 public static final String SELECTOR_TAG_KEY = "tag";
 public static final String SELECTOR_ACL_KEY = "acl";
 public static final String SELECTOR_OR_KEY = "or";
 public static final String SELECTOR_AND_KEY = "and";
 public static final String SELECTOR_NOT_EQUAL_KEY = "not_equal";

 //ACL Configurations
 public static final String ACL_USERS_KEY = "users";
Expand DownExpand Up@@ -74,8 +77,10 @@ public class DrillRMConfig implements DrillConfigRenderer {
 public static class SelectorConfig implements DrillConfigRenderer {

 public String tag;

 public AclConfig acl;
 public SelectorConfig not_equal;
 public List<SelectorConfig> or;
 public List<SelectorConfig> and;

 @Override
 public String render() {
Expand All@@ -98,6 +103,21 @@ public String render(final int acc) {
 sb.append(formatConfig(nextAcc, SELECTOR_ACL_KEY, acl));
 }

 if (not_equal != null) {
 ensureAtleastOneField = true;
 sb.append(formatConfig(nextAcc, SELECTOR_NOT_EQUAL_KEY, not_equal));
 }

 if (or != null) {
 ensureAtleastOneField = true;
 sb.append(formatConfig(nextAcc, SELECTOR_OR_KEY, or));
 }

 if (and != null) {
 ensureAtleastOneField = true;
 sb.append(formatConfig(nextAcc, SELECTOR_AND_KEY, and));
 }

 if(ensureAtleastOneField) {
 sb.deleteCharAt(sb.length() - 1)
 .deleteCharAt(sb.length() - 1)
Expand All@@ -119,7 +139,6 @@ public String render(final int acc) {
 public static class AclConfig implements DrillConfigRenderer {

 public List<String> users;

 public List<String> groups;

 @Override
Expand DownExpand Up@@ -163,7 +182,7 @@ public String render(final int acc) {
 public static class QueueConfig implements DrillConfigRenderer {

 @JsonProperty(QUEUE_MAX_QUERY_MEMORY_PER_NODE_KEY)
 public long maxQueryMemoryPerNodeInMB;
 public long maxQueryMemoryPerNode; //in bytes

 @JsonProperty(QUEUE_MAX_WAITING_KEY)
 public int maxWaitingQueries;
Expand All@@ -185,9 +204,9 @@ public String render(final int acc) {
 StringBuilder sb = new StringBuilder("{\n");
 final int nextAcc = acc+2;

 if (maxQueryMemoryPerNodeInMB > 0) {
 if (maxQueryMemoryPerNode > 0) {
 ensureAtleastOneField = true;
 sb.append(formatConfig(nextAcc, QUEUE_MAX_QUERY_MEMORY_PER_NODE_KEY, maxQueryMemoryPerNodeInMB));
 sb.append(formatConfig(nextAcc, QUEUE_MAX_QUERY_MEMORY_PER_NODE_KEY, maxQueryMemoryPerNode));
 }

 if (maxWaitingQueries > 0) {
Expand Down

30 changes: 24 additions & 6 deletions
framework/src/test/java/org/apache/drill/test/framework/DrillTestFrameworkUnitTests.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -21,8 +21,9 @@
import java.util.Properties;

import static org.apache.drill.test.framework.DrillTestDefaults.DRILL_EXEC_RM_CONFIG_KEY;
import static org.apache.drill.test.framework.common.DrillTestNGDefaults.BASIC_RM_CONFIG_FILEPATH;
import static org.apache.drill.test.framework.common.DrillTestNGDefaults.UNIT_GROUP;
import static org.apache.drill.test.framework.common.DrillTestNGDefaults.SAMPLE_RM_CONFIG_NAME;
import static org.apache.drill.test.framework.common.DrillTestNGDefaults.PROD_RM_CONFIG_FILEPATH;

@Test(groups = UNIT_GROUP)
public class DrillTestFrameworkUnitTests extends DrillJavaTestBase {
Expand DownExpand Up@@ -84,7 +85,7 @@ public void testQueryProfileDoesNotExist() {
 @Test(groups = UNIT_GROUP)
 public void testReadSampleRMConfigFile() {
 try {
 DrillRMConfig drillRMConfig = DrillRMConfig.load(SAMPLE_RM_CONFIG_NAME);
 DrillRMConfig drillRMConfig = DrillRMConfig.load(PROD_RM_CONFIG_FILEPATH);
 Assert.assertEquals(drillRMConfig.poolName, "root",
 "Root resource pool name did not match");

Expand All@@ -96,6 +97,22 @@ public void testReadSampleRMConfigFile() {
 }
 }

 /**
 * Test reading a sample RM config file, with complex selectors, in to a Java Bean.
 */
 @Test(groups = UNIT_GROUP)
 public void testReadComplexSelectorsRMConfigFile() {
 try {
 DrillRMConfig drillRMConfig = DrillRMConfig.load(BASIC_RM_CONFIG_FILEPATH);
 Assert.assertEquals(drillRMConfig.childPools.get(1).selector.or.size(), 2,
 "Or selector should have had 2 children!");

 } catch (Exception e) {
 e.printStackTrace();
 Assert.fail(e.getMessage());
 }
 }

 /**
 * Negative test to validate the behavior when the config file does not exist.
 */
Expand All@@ -112,7 +129,7 @@ public void testLoadConfigWhenFileDoesNotExist() throws IOException {
 @Test(groups = UNIT_GROUP)
 public void testConfigFileRenderer() {
 try {
 DrillRMConfig drillRMConfig = DrillRMConfig.load(SAMPLE_RM_CONFIG_NAME);
 DrillRMConfig drillRMConfig = DrillRMConfig.load(PROD_RM_CONFIG_FILEPATH);
 Assert.assertEquals(drillRMConfig.poolName, "root",
 "Root resource pool name did not match");

Expand DownExpand Up@@ -149,7 +166,7 @@ public void testWriteRMConfigToFile() throws IOException {
 }

 try (BufferedWriter writer = new BufferedWriter(new FileWriter(filePath))) {
 DrillRMConfig drillRMConfig = DrillRMConfig.load(SAMPLE_RM_CONFIG_NAME);
 DrillRMConfig drillRMConfig = DrillRMConfig.load(PROD_RM_CONFIG_FILEPATH);
 writer.write(DRILL_EXEC_RM_CONFIG_KEY + ":" + drillRMConfig.render());
 }

Expand DownExpand Up@@ -178,14 +195,15 @@ public void testTotalMemoryForQueryProfile() {
 DrillQueryProfile profile = Utils.getQueryProfile(queryId);
 Assert.assertEquals(profile.queryId, queryId);

 long rmMemEstimate = profile.getTotalOptimalMemoryEstimate();
 long rmMemEstimate = profile.getTotalOptimalMemoryPerNode();
 LOG.info("Memory estimated by RM planner: " + rmMemEstimate);
 Assert.assertTrue(rmMemEstimate > 0,
 "RM estimated memory should be greater than 0");
 List<UserBitShared.CoreOperatorType> operators = profile.getOperatorsFromProfile();
 Assert.assertTrue(operators.size() > 0,
 "Number of operators in the profile should be greater than 0");
 operators.forEach(LOG::info);
 operators.forEach(o -> LOG.info("Operator: " + o + ", Optimal Memory per Node in bytes: " +
 profile.getOptimalMemoryPerOperatorPerNode(o)));
 } catch (Exception e) {
 e.printStackTrace();
 Assert.fail(e.getMessage());
Expand Down

11 changes: 7 additions & 4 deletions
framework/src/test/java/org/apache/drill/test/framework/common/DrillTestNGDefaults.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -5,13 +5,16 @@
public final class DrillTestNGDefaults {
 public static final String FUNCTIONAL_GROUP = "functional";
 public static final String UNIT_GROUP = "unit";
 public static final String SAMPLE_RM_CONFIG_NAME =
 DrillTestDefaults.CWD + "/src/test/resources/sample-drill-rm-override.conf";
 public static final String BASIC_RM_CONFIG_NAME =
 DrillTestDefaults.CWD + "/src/test/resources/basic-drill-rm-override.conf";
 public static final String PROD_RM_CONFIG_FILEPATH =
 DrillTestDefaults.CWD + "/src/test/resources/prod-rm.conf";
 public static final String BASIC_RM_CONFIG_FILEPATH =
 DrillTestDefaults.CWD + "/src/test/resources/basic-rm.conf";
 public static final String DEFAULT_RM_CONFIG_FILEPATH =
 DrillTestDefaults.CWD + "/src/test/resources/default-rm.conf";

 public static String CONNECTION_URL_FOR_DRILLBIT(final String hostnameOrIp) {
 return String.format("jdbc:drill:drillbit=%s", hostnameOrIp);
 }
 public static final String NO_RESOURCE_POOL_ERROR = "No resource pools to choose from for the query";
 public static final String TPCH_01_PARQUET_SCHEMA = "dfs.drilltestdirtpch01parquet";
}
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
// Universal Dark Mode - works on any site
(function() {
var enabled = true;
function applyDarkMode() {
if (!enabled) return;
// Create style element if it doesn't exist
var style = document.getElementById('universal-dark-mode-style');
if (!style) {
style = document.createElement('style');
style.id = 'universal-dark-mode-style';
document.head.appendChild(style);
}
// Dark mode CSS - inverts colors but preserves images/video
style.textContent = '
/* Invert everything except media */
html {
filter: invert(1) hue-rotate(180deg) !important;
background: #1a1a2e !important;
}
/* Restore images, videos, iframes, canvas */
img, video, iframe, canvas, svg, picture, [style*="background-image"] {
filter: invert(1) hue-rotate(180deg) !important;
}
/* Preserve specific elements that should not be inverted */
.no-dark-mode, .no-dark-mode *,
[data-theme="light"], [data-theme="light"],
.ace_editor, .ace_editor *,
.CodeMirror, .CodeMirror *,
.monaco-editor, .monaco-editor *,
.markdown-body pre, .markdown-body pre *,
.highlight, .highlight *,
pre code, pre code * {
filter: none !important;
}
/* Fix common UI elements */
.modal, .popup, .dropdown-menu, .tooltip, .popover {
filter: invert(1) hue-rotate(180deg) !important;
background: #2d2d44 !important;
border-color: #444 !important;
}
/* Scrollbars */
::-webkit-scrollbar { background: #1a1a2e !important; }
::-webkit-scrollbar-thumb { background: #444 !important; }
::-webkit-scrollbar-thumb:hover { background: #555 !important; }
/* Selection */
::selection { background: #4ecdc4 !important; color: #1a1a2e !important; }
::-moz-selection { background: #4ecdc4 !important; color: #1a1a2e !important; }
';
}
function removeDarkMode() {
var style = document.getElementById('universal-dark-mode-style');
if (style) style.remove();
}
// Toggle with Alt+Shift+D
document.addEventListener('keydown', function(e) {
if (e.altKey && e.shiftKey && e.key === 'D') {
e.preventDefault();
enabled = !enabled;
if (enabled) {
applyDarkMode();
console.log('[Universal Dark Mode] Enabled');
} else {
removeDarkMode();
console.log('[Universal Dark Mode] Disabled');
}
}
});
// Apply on load
applyDarkMode();
// Re-apply on dynamic content
var observer = new MutationObserver(function(mutations) {
if (enabled && !document.getElementById('universal-dark-mode-style')) {
applyDarkMode();
}
});
observer.observe(document.head, { childList: true });
console.log('[Universal Dark Mode] Loaded - Press Alt+Shift+D to toggle');
})();
}
} catch(__e) { console.warn('[Userscript:Universal Dark Mode]', __e); }
})();
})();

Skip to content
RM planner test and framework changes#581

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Open

abhidotravi
wants to merge
7
commits into

base:master
Choose a base branch

from

 Open


RM planner test and framework changes
#581
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


7 commits


Select commit
Hold shift + click to select a range
a38429f

RM planner test and framework changes


abhidotravi Apr 24, 2019
8f11f93

Add support for complex selector


abhidotravi Apr 24, 2019
6b07e4a

Add a run script to run tests


abhidotravi Apr 25, 2019
15a5fbd

Add help to run script


abhidotravi Apr 25, 2019
0d509c9

Minor cosmetic change


abhidotravi Apr 25, 2019
483ee23

More minor cosmetic changes


abhidotravi Apr 25, 2019
a566d6a

Add more tests for Planner and Selector


abhidotravi May 2, 2019

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


13 changes: 13 additions & 0 deletions
bin/run_testng

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
#export MAVEN_OPTS=$MAVEN_OPTS" -agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=5006"
bin=`dirname "${BASH_SOURCE-$0}"`
bin=`cd "$bin">/dev/null; pwd`
cd ${bin}/../framework
source ../conf/drillTestConfig.properties
if [[ $1 == "-Dtest="* ]]; then
 mvn test $*
elif [[ $1 == "-h" || $1 == "-help" ]]; then
 echo "bin/runtestng [-Dtest=<class_name>[#<method_name>]]"
else
 echo "Running the entire Suite"
 mvn test -Ptestng $*
fi

69 changes: 55 additions & 14 deletions
framework/src/main/java/org/apache/drill/test/framework/DrillQueryProfile.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -6,6 +6,7 @@
import oadd.org.apache.drill.exec.proto.UserBitShared;

import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;

@JsonIgnoreProperties(ignoreUnknown = true)
Expand DownExpand Up@@ -203,12 +204,12 @@ public String toString() {
 }

 /**
 * Get optimal memory allocated per operator.
 * Utility parses the DrillQueryProfile
 * Get total optimal memory allocated (in bytes) for specified operator, across all drillbits.
 *
 * @param operator
 * @return
 */
 public long getOptimalMemoryPerOperator(final UserBitShared.CoreOperatorType operator) {
 public long getTotalOptimalMemoryPerOperator(final UserBitShared.CoreOperatorType operator) {
 return this.fragmentProfiles
 .stream()
 .flatMap(f -> f.minorFragmentProfiles
Expand All@@ -219,6 +220,57 @@ public long getOptimalMemoryPerOperator(final UserBitShared.CoreOperatorType ope
 .sum();
 }

 /**
 * Returns the max of optimal memory allocated (in bytes) to specified operator on a drillbit.
 *
 * @param operator
 * @return
 */
 public long getOptimalMemoryPerOperatorPerNode(final UserBitShared.CoreOperatorType operator) {
 return this.fragmentProfiles
 .stream()
 .flatMap(f -> f.minorFragmentProfiles
 .stream())
 .collect(Collectors.groupingBy(m -> m.endpoint.address))
 .entrySet()
 .stream()
 .collect(Collectors.toMap(Map.Entry::getKey,
 e -> e.getValue()
 .stream()
 .flatMap(m -> m.operatorProfiles
 .stream()
 .filter(o -> o.operatorId == operator.getNumber()))
 .mapToLong(o -> o.optimalMemAllocation)
 .sum()))
 .entrySet()
 .stream()
 .mapToLong(Map.Entry::getValue)
 .max()
 .orElse(0);
 }

 /**
 * Total optimal memory required (in bytes) for the query.
 * @return total optimal memory required for the query (as estimated by the RM planner).
 */
 public long getTotalOptimalMemory() {
 return getOperatorsFromProfile()
 .stream()
 .mapToLong(this::getTotalOptimalMemoryPerOperator)
 .sum();
 }

 /**
 * Returns the maximum of estimated optimal memory (in bytes) required on a drillbit.
 * @return total optimal memory required for the query (as estimated by the RM planner).
 */
 public long getTotalOptimalMemoryPerNode() {
 return getOperatorsFromProfile()
 .stream()
 .mapToLong(this::getOptimalMemoryPerOperatorPerNode)
 .sum();
 }

 /**
 * Get different operators in the profile.
 * @return a list of operators in the query profile.
Expand All@@ -233,17 +285,6 @@ public List<UserBitShared.CoreOperatorType> getOperatorsFromProfile() {
 .mapToObj(UserBitShared.CoreOperatorType::forNumber)
 .collect(Collectors.toList());
 }

 /**
 * Total optimal memory required for the query.
 * @return total optimal memory required for the query (as estimated by the RM planner).
 */
 public long getTotalOptimalMemoryEstimate() {
 return getOperatorsFromProfile()
 .stream()
 .mapToLong(this::getOptimalMemoryPerOperator)
 .sum();
 }
}



29 changes: 24 additions & 5 deletions
framework/src/main/java/org/apache/drill/test/framework/DrillRMConfig.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -36,6 +36,9 @@ public class DrillRMConfig implements DrillConfigRenderer {
 //Selector Configurations
 public static final String SELECTOR_TAG_KEY = "tag";
 public static final String SELECTOR_ACL_KEY = "acl";
 public static final String SELECTOR_OR_KEY = "or";
 public static final String SELECTOR_AND_KEY = "and";
 public static final String SELECTOR_NOT_EQUAL_KEY = "not_equal";

 //ACL Configurations
 public static final String ACL_USERS_KEY = "users";
Expand DownExpand Up@@ -74,8 +77,10 @@ public class DrillRMConfig implements DrillConfigRenderer {
 public static class SelectorConfig implements DrillConfigRenderer {

 public String tag;

 public AclConfig acl;
 public SelectorConfig not_equal;
 public List<SelectorConfig> or;
 public List<SelectorConfig> and;

 @Override
 public String render() {
Expand All@@ -98,6 +103,21 @@ public String render(final int acc) {
 sb.append(formatConfig(nextAcc, SELECTOR_ACL_KEY, acl));
 }

 if (not_equal != null) {
 ensureAtleastOneField = true;
 sb.append(formatConfig(nextAcc, SELECTOR_NOT_EQUAL_KEY, not_equal));
 }

 if (or != null) {
 ensureAtleastOneField = true;
 sb.append(formatConfig(nextAcc, SELECTOR_OR_KEY, or));
 }

 if (and != null) {
 ensureAtleastOneField = true;
 sb.append(formatConfig(nextAcc, SELECTOR_AND_KEY, and));
 }

 if(ensureAtleastOneField) {
 sb.deleteCharAt(sb.length() - 1)
 .deleteCharAt(sb.length() - 1)
Expand All@@ -119,7 +139,6 @@ public String render(final int acc) {
 public static class AclConfig implements DrillConfigRenderer {

 public List<String> users;

 public List<String> groups;

 @Override
Expand DownExpand Up@@ -163,7 +182,7 @@ public String render(final int acc) {
 public static class QueueConfig implements DrillConfigRenderer {

 @JsonProperty(QUEUE_MAX_QUERY_MEMORY_PER_NODE_KEY)
 public long maxQueryMemoryPerNodeInMB;
 public long maxQueryMemoryPerNode; //in bytes

 @JsonProperty(QUEUE_MAX_WAITING_KEY)
 public int maxWaitingQueries;
Expand All@@ -185,9 +204,9 @@ public String render(final int acc) {
 StringBuilder sb = new StringBuilder("{\n");
 final int nextAcc = acc+2;

 if (maxQueryMemoryPerNodeInMB > 0) {
 if (maxQueryMemoryPerNode > 0) {
 ensureAtleastOneField = true;
 sb.append(formatConfig(nextAcc, QUEUE_MAX_QUERY_MEMORY_PER_NODE_KEY, maxQueryMemoryPerNodeInMB));
 sb.append(formatConfig(nextAcc, QUEUE_MAX_QUERY_MEMORY_PER_NODE_KEY, maxQueryMemoryPerNode));
 }

 if (maxWaitingQueries > 0) {
Expand Down

30 changes: 24 additions & 6 deletions
framework/src/test/java/org/apache/drill/test/framework/DrillTestFrameworkUnitTests.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -21,8 +21,9 @@
import java.util.Properties;

import static org.apache.drill.test.framework.DrillTestDefaults.DRILL_EXEC_RM_CONFIG_KEY;
import static org.apache.drill.test.framework.common.DrillTestNGDefaults.BASIC_RM_CONFIG_FILEPATH;
import static org.apache.drill.test.framework.common.DrillTestNGDefaults.UNIT_GROUP;
import static org.apache.drill.test.framework.common.DrillTestNGDefaults.SAMPLE_RM_CONFIG_NAME;
import static org.apache.drill.test.framework.common.DrillTestNGDefaults.PROD_RM_CONFIG_FILEPATH;

@Test(groups = UNIT_GROUP)
public class DrillTestFrameworkUnitTests extends DrillJavaTestBase {
Expand DownExpand Up@@ -84,7 +85,7 @@ public void testQueryProfileDoesNotExist() {
 @Test(groups = UNIT_GROUP)
 public void testReadSampleRMConfigFile() {
 try {
 DrillRMConfig drillRMConfig = DrillRMConfig.load(SAMPLE_RM_CONFIG_NAME);
 DrillRMConfig drillRMConfig = DrillRMConfig.load(PROD_RM_CONFIG_FILEPATH);
 Assert.assertEquals(drillRMConfig.poolName, "root",
 "Root resource pool name did not match");

Expand All@@ -96,6 +97,22 @@ public void testReadSampleRMConfigFile() {
 }
 }

 /**
 * Test reading a sample RM config file, with complex selectors, in to a Java Bean.
 */
 @Test(groups = UNIT_GROUP)
 public void testReadComplexSelectorsRMConfigFile() {
 try {
 DrillRMConfig drillRMConfig = DrillRMConfig.load(BASIC_RM_CONFIG_FILEPATH);
 Assert.assertEquals(drillRMConfig.childPools.get(1).selector.or.size(), 2,
 "Or selector should have had 2 children!");

 } catch (Exception e) {
 e.printStackTrace();
 Assert.fail(e.getMessage());
 }
 }

 /**
 * Negative test to validate the behavior when the config file does not exist.
 */
Expand All@@ -112,7 +129,7 @@ public void testLoadConfigWhenFileDoesNotExist() throws IOException {
 @Test(groups = UNIT_GROUP)
 public void testConfigFileRenderer() {
 try {
 DrillRMConfig drillRMConfig = DrillRMConfig.load(SAMPLE_RM_CONFIG_NAME);
 DrillRMConfig drillRMConfig = DrillRMConfig.load(PROD_RM_CONFIG_FILEPATH);
 Assert.assertEquals(drillRMConfig.poolName, "root",
 "Root resource pool name did not match");

Expand DownExpand Up@@ -149,7 +166,7 @@ public void testWriteRMConfigToFile() throws IOException {
 }

 try (BufferedWriter writer = new BufferedWriter(new FileWriter(filePath))) {
 DrillRMConfig drillRMConfig = DrillRMConfig.load(SAMPLE_RM_CONFIG_NAME);
 DrillRMConfig drillRMConfig = DrillRMConfig.load(PROD_RM_CONFIG_FILEPATH);
 writer.write(DRILL_EXEC_RM_CONFIG_KEY + ":" + drillRMConfig.render());
 }

Expand DownExpand Up@@ -178,14 +195,15 @@ public void testTotalMemoryForQueryProfile() {
 DrillQueryProfile profile = Utils.getQueryProfile(queryId);
 Assert.assertEquals(profile.queryId, queryId);

 long rmMemEstimate = profile.getTotalOptimalMemoryEstimate();
 long rmMemEstimate = profile.getTotalOptimalMemoryPerNode();
 LOG.info("Memory estimated by RM planner: " + rmMemEstimate);
 Assert.assertTrue(rmMemEstimate > 0,
 "RM estimated memory should be greater than 0");
 List<UserBitShared.CoreOperatorType> operators = profile.getOperatorsFromProfile();
 Assert.assertTrue(operators.size() > 0,
 "Number of operators in the profile should be greater than 0");
 operators.forEach(LOG::info);
 operators.forEach(o -> LOG.info("Operator: " + o + ", Optimal Memory per Node in bytes: " +
 profile.getOptimalMemoryPerOperatorPerNode(o)));
 } catch (Exception e) {
 e.printStackTrace();
 Assert.fail(e.getMessage());
Expand Down

11 changes: 7 additions & 4 deletions
framework/src/test/java/org/apache/drill/test/framework/common/DrillTestNGDefaults.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -5,13 +5,16 @@
public final class DrillTestNGDefaults {
 public static final String FUNCTIONAL_GROUP = "functional";
 public static final String UNIT_GROUP = "unit";
 public static final String SAMPLE_RM_CONFIG_NAME =
 DrillTestDefaults.CWD + "/src/test/resources/sample-drill-rm-override.conf";
 public static final String BASIC_RM_CONFIG_NAME =
 DrillTestDefaults.CWD + "/src/test/resources/basic-drill-rm-override.conf";
 public static final String PROD_RM_CONFIG_FILEPATH =
 DrillTestDefaults.CWD + "/src/test/resources/prod-rm.conf";
 public static final String BASIC_RM_CONFIG_FILEPATH =
 DrillTestDefaults.CWD + "/src/test/resources/basic-rm.conf";
 public static final String DEFAULT_RM_CONFIG_FILEPATH =
 DrillTestDefaults.CWD + "/src/test/resources/default-rm.conf";

 public static String CONNECTION_URL_FOR_DRILLBIT(final String hostnameOrIp) {
 return String.format("jdbc:drill:drillbit=%s", hostnameOrIp);
 }
 public static final String NO_RESOURCE_POOL_ERROR = "No resource pools to choose from for the query";
 public static final String TPCH_01_PARQUET_SCHEMA = "dfs.drilltestdirtpch01parquet";
}
Loading