Skip to content

(Refactor)[Nereids] Combine operator and plan#10786

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
EmmyMiao87 merged 8 commits into from
Jul 13, 2022
Merged

(Refactor)[Nereids] Combine operator and plan #10786

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
8af696f
combine op and plan
924060929 Jul 12, 2022
6877045
fix ut
924060929 Jul 12, 2022
ddaa70d
rename Logical/Physical NaryPlan to Logical/Physical Nary
924060929 Jul 12, 2022
0cfdba4
fix translator
924060929 Jul 12, 2022
0ec1c82
replace operator to plan
924060929 Jul 12, 2022
8157baa
add getPhysicalProperties method to PhysicalPlan
924060929 Jul 13, 2022
abc66cb
fix bug
924060929 Jul 13, 2022
acabe12
fix
924060929 Jul 13, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion fe/fe-core/pom.xml
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -859,7 +859,7 @@ under the License.
<configuration>
<proc>only</proc>
<compilerArgs>
<arg>-AoperatorPath=${basedir}/src/main/java/org/apache/doris/nereids</arg>
<arg>-AplanPath=${basedir}/src/main/java/org/apache/doris/nereids</arg>
</compilerArgs>
<includes>
<include>org/apache/doris/nereids/pattern/generator/PatternDescribableProcessPoint.java</include>
Expand Down
1 change: 1 addition & 0 deletions fe/fe-core/src/main/java/org/apache/doris/analysis/BuiltinAggregateFunction.java
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -127,6 +127,7 @@ public enum Operator {
// The intermediate type for this function if it is constant regardless of
// input type. Set to null if it can only be determined during analysis.
private final org.apache.doris.catalog.Type intermediateType;

Operator(String description, TAggregationOp thriftOp,
org.apache.doris.catalog.Type intermediateType) {
this.description = description;
Expand Down
3 changes: 2 additions & 1 deletion fe/fe-core/src/main/java/org/apache/doris/analysis/DateLiteral.java
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -158,7 +158,8 @@ private enum DateLiteralType {
DATEV2(3);

private final int value;
private DateLiteralType(int value) {

DateLiteralType(int value) {

@morrySnowmorrySnowJul 12, 2022

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

why do this?

@924060929924060929Jul 13, 2022

Copy link
Copy Markdown
ContributorAuthor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

fix checkstyle. and private enum DateLiteralType's constructor is also private, so can be omitted

this.value = value;
}

Expand Down
3 changes: 2 additions & 1 deletion fe/fe-core/src/main/java/org/apache/doris/common/InternalErrorCode.java
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -37,7 +37,8 @@ public enum InternalErrorCode {
TASKS_ABORT_ERR(104);

private long errCode;
private InternalErrorCode(long code) {

InternalErrorCode(long code) {
this.errCode = code;
}

Expand Down
3 changes: 1 addition & 2 deletions fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -150,8 +150,7 @@ private PhysicalPlan chooseBestPlan(Group rootGroup, PhysicalProperties physical
planChildren.add(chooseBestPlan(groupExpression.child(i), inputPropertiesList.get(i)));
}

Plan plan = ((PhysicalPlan) groupExpression.getOperator().toTreeNode(groupExpression)).withChildren(
planChildren);
Plan plan = groupExpression.getPlan().withChildren(planChildren);
if (!(plan instanceof PhysicalPlan)) {
throw new AnalysisException("generate logical plan");
}
Expand Down
4 changes: 2 additions & 2 deletions fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundAlias.java
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -18,8 +18,8 @@
package org.apache.doris.nereids.analyzer;

import org.apache.doris.nereids.exceptions.UnboundException;
import org.apache.doris.nereids.trees.NodeType;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.ExpressionType;
import org.apache.doris.nereids.trees.expressions.NamedExpression;
import org.apache.doris.nereids.trees.expressions.UnaryExpression;
import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor;
Expand All@@ -35,7 +35,7 @@
public class UnboundAlias extends NamedExpression implements UnaryExpression, Unbound {

public UnboundAlias(Expression child) {
super(NodeType.UNBOUND_ALIAS, child);
super(ExpressionType.UNBOUND_ALIAS, child);
}

@Override
Expand Down
4 changes: 2 additions & 2 deletions fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundFunction.java
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -18,8 +18,8 @@
package org.apache.doris.nereids.analyzer;

import org.apache.doris.nereids.exceptions.UnboundException;
import org.apache.doris.nereids.trees.NodeType;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.ExpressionType;
import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor;

import com.google.common.base.Joiner;
Expand All@@ -37,7 +37,7 @@ public class UnboundFunction extends Expression implements Unbound {
private final boolean isDistinct;

public UnboundFunction(String name, boolean isDistinct, List<Expression> arguments) {
super(NodeType.UNBOUND_FUNCTION, arguments.toArray(new Expression[0]));
super(ExpressionType.UNBOUND_FUNCTION, arguments.toArray(new Expression[0]));
this.name = Objects.requireNonNull(name, "name can not be null");
this.isDistinct = isDistinct;
}
Expand Down
40 changes: 34 additions & 6 deletions fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundRelation.java
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -19,38 +19,51 @@

import org.apache.doris.nereids.analyzer.identifier.TableIdentifier;
import org.apache.doris.nereids.exceptions.UnboundException;
import org.apache.doris.nereids.operators.OperatorType;
import org.apache.doris.nereids.operators.plans.logical.LogicalLeafOperator;
import org.apache.doris.nereids.memo.GroupExpression;
import org.apache.doris.nereids.properties.LogicalProperties;
import org.apache.doris.nereids.properties.UnboundLogicalProperties;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.Slot;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.PlanType;
import org.apache.doris.nereids.trees.plans.logical.LogicalLeaf;
import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor;
import org.apache.doris.nereids.util.Utils;

import com.google.common.collect.Lists;
import org.apache.commons.lang3.StringUtils;

import java.util.List;
import java.util.Optional;

/**
* Represent a relation plan node that has not been bound.
*/
public class UnboundRelation extends LogicalLeafOperator implements Unbound {
public class UnboundRelation extends LogicalLeaf implements Unbound {
private final List<String> nameParts;

public UnboundRelation(List<String> nameParts) {
super(OperatorType.LOGICAL_UNBOUND_RELATION);
this(nameParts, Optional.empty(), Optional.empty());
}

public UnboundRelation(List<String> nameParts, Optional<GroupExpression> groupExpression,
Optional<LogicalProperties> logicalProperties) {
super(PlanType.LOGICAL_UNBOUND_RELATION, groupExpression, logicalProperties);
this.nameParts = nameParts;
}

public UnboundRelation(TableIdentifier identifier) {
this(identifier, Optional.empty(), Optional.empty());
}

/**
* Constructor for UnboundRelation.
*
* @param identifier relation identifier
*/
public UnboundRelation(TableIdentifier identifier) {
super(OperatorType.LOGICAL_UNBOUND_RELATION);
public UnboundRelation(TableIdentifier identifier, Optional<GroupExpression> groupExpression,
Optional<LogicalProperties> logicalProperties) {
super(PlanType.LOGICAL_UNBOUND_RELATION, groupExpression, logicalProperties);
this.nameParts = Lists.newArrayList();
if (identifier.getDatabaseName().isPresent()) {
nameParts.add(identifier.getDatabaseName().get());
Expand All@@ -72,6 +85,16 @@ public LogicalProperties computeLogicalProperties(Plan... inputs) {
return new UnboundLogicalProperties();
}

@Override
public Plan withGroupExpression(Optional<GroupExpression> groupExpression) {
return new UnboundRelation(nameParts, groupExpression, Optional.of(logicalProperties));
}

@Override
public Plan withLogicalProperties(Optional<LogicalProperties> logicalProperties) {
return new UnboundRelation(nameParts, Optional.empty(), logicalProperties);

@morrySnowmorrySnowJul 12, 2022

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

groupexpression should use original one？

@924060929924060929Jul 13, 2022

Copy link
Copy Markdown
ContributorAuthor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

when the plan is changed, the groupExpression must be clear to tell the memo: I create a new plan, don't skip copyIn.

}

@Override
public List<Slot> computeOutput() {
throw new UnboundException("output");
Expand All@@ -82,6 +105,11 @@ public String toString() {
return "UnresolvedRelation" + "(" + StringUtils.join(nameParts, ".") + ")";
}

@Override
public <R, C> R accept(PlanVisitor<R, C> visitor, C context) {
return visitor.visitUnboundRelation(this, context);
}

@Override
public List<Expression> getExpressions() {
throw new UnsupportedOperationException(this.getClass().getSimpleName() + " don't support getExpression()");
Expand Down
4 changes: 2 additions & 2 deletions fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundSlot.java
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -17,7 +17,7 @@

package org.apache.doris.nereids.analyzer;

import org.apache.doris.nereids.trees.NodeType;
import org.apache.doris.nereids.trees.expressions.ExpressionType;
import org.apache.doris.nereids.trees.expressions.Slot;
import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor;
import org.apache.doris.nereids.util.Utils;
Expand All@@ -33,7 +33,7 @@ public class UnboundSlot extends Slot implements Unbound {
private final List<String> nameParts;

public UnboundSlot(List<String> nameParts) {
super(NodeType.UNBOUND_SLOT);
super(ExpressionType.UNBOUND_SLOT);
this.nameParts = nameParts;
}

Expand Down
4 changes: 2 additions & 2 deletions fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundStar.java
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -18,7 +18,7 @@
package org.apache.doris.nereids.analyzer;

import org.apache.doris.nereids.exceptions.UnboundException;
import org.apache.doris.nereids.trees.NodeType;
import org.apache.doris.nereids.trees.expressions.ExpressionType;
import org.apache.doris.nereids.trees.expressions.LeafExpression;
import org.apache.doris.nereids.trees.expressions.NamedExpression;
import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor;
Expand All@@ -35,7 +35,7 @@ public class UnboundStar extends NamedExpression implements LeafExpression, Unbo
private final List<String> qualifier;

public UnboundStar(List<String> qualifier) {
super(NodeType.UNBOUND_STAR);
super(ExpressionType.UNBOUND_STAR);
this.qualifier = qualifier;
}

Expand Down
22 changes: 11 additions & 11 deletions fe/fe-core/src/main/java/org/apache/doris/nereids/cost/CostCalculator.java
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -20,12 +20,12 @@
import org.apache.doris.common.Id;
import org.apache.doris.nereids.PlanContext;
import org.apache.doris.nereids.memo.GroupExpression;
import org.apache.doris.nereids.operators.Operator;
import org.apache.doris.nereids.operators.OperatorVisitor;
import org.apache.doris.nereids.operators.plans.physical.PhysicalAggregate;
import org.apache.doris.nereids.operators.plans.physical.PhysicalHashJoin;
import org.apache.doris.nereids.operators.plans.physical.PhysicalOlapScan;
import org.apache.doris.nereids.operators.plans.physical.PhysicalProject;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.physical.PhysicalAggregate;
import org.apache.doris.nereids.trees.plans.physical.PhysicalHashJoin;
import org.apache.doris.nereids.trees.plans.physical.PhysicalOlapScan;
import org.apache.doris.nereids.trees.plans.physical.PhysicalProject;
import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor;
import org.apache.doris.statistics.StatsDeriveResult;

import com.google.common.base.Preconditions;
Expand All@@ -43,7 +43,7 @@ public class CostCalculator {
public static double calculateCost(GroupExpression groupExpression) {
PlanContext planContext = new PlanContext(groupExpression);
CostEstimator costCalculator = new CostEstimator();
CostEstimate costEstimate = groupExpression.getOperator().accept(costCalculator, planContext);
CostEstimate costEstimate = groupExpression.getPlan().accept(costCalculator, planContext);
return costFormula(costEstimate);
}

Expand All@@ -55,14 +55,14 @@ private static double costFormula(CostEstimate costEstimate) {
+ costEstimate.getNetworkCost() * networkCostWeight;
}

private static class CostEstimator extends OperatorVisitor<CostEstimate, PlanContext> {
private static class CostEstimator extends PlanVisitor<CostEstimate, PlanContext> {
@Override
public CostEstimate visitOperator(Operator operator, PlanContext context) {
public CostEstimate visit(Plan plan, PlanContext context) {
return CostEstimate.zero();
}

@Override
public CostEstimate visitPhysicalAggregation(PhysicalAggregate physicalAggregate, PlanContext context) {
public CostEstimate visitPhysicalAggregate(PhysicalAggregate<Plan> aggregate, PlanContext context) {
StatsDeriveResult statistics = context.getStatisticsWithCheck();
return CostEstimate.ofCpu(statistics.computeSize());
}
Expand All@@ -74,7 +74,7 @@ public CostEstimate visitPhysicalOlapScan(PhysicalOlapScan physicalOlapScan, Pla
}

@Override
public CostEstimate visitPhysicalHashJoin(PhysicalHashJoin physicalHashJoin, PlanContext context) {
public CostEstimate visitPhysicalHashJoin(PhysicalHashJoin<Plan, Plan> physicalHashJoin, PlanContext context) {
Preconditions.checkState(context.getGroupExpression().arity() == 2);
Preconditions.checkState(context.getChildrenStats().size() == 2);

Expand Down
4 changes: 2 additions & 2 deletions fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/ExpressionTranslator.java
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -29,13 +29,13 @@
import org.apache.doris.analysis.StringLiteral;
import org.apache.doris.catalog.Type;
import org.apache.doris.nereids.exceptions.AnalysisException;
import org.apache.doris.nereids.trees.NodeType;
import org.apache.doris.nereids.trees.expressions.Alias;
import org.apache.doris.nereids.trees.expressions.Arithmetic;
import org.apache.doris.nereids.trees.expressions.Between;
import org.apache.doris.nereids.trees.expressions.CompoundPredicate;
import org.apache.doris.nereids.trees.expressions.EqualTo;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.ExpressionType;
import org.apache.doris.nereids.trees.expressions.GreaterThan;
import org.apache.doris.nereids.trees.expressions.GreaterThanEqual;
import org.apache.doris.nereids.trees.expressions.LessThan;
Expand DownExpand Up@@ -170,7 +170,7 @@ public Expr visitBetween(Between between, PlanTranslatorContext context) {

@Override
public Expr visitCompoundPredicate(CompoundPredicate compoundPredicate, PlanTranslatorContext context) {
NodeType nodeType = compoundPredicate.getType();
ExpressionType nodeType = compoundPredicate.getType();
org.apache.doris.analysis.CompoundPredicate.Operator staleOp;
switch (nodeType) {
case OR:
Expand Down
Loading