apache · morrySnow · Aug 4, 2026 · Jul 16, 2026 · Jul 17, 2026 · Jul 17, 2026
@@ -107,7 +107,8 @@ protected Optional<CopyInResult> invokeRewriteRuleWithTrace(Rule rule, Plan befo
 
  CopyInResult result = context.getCascadesContext()
  .getMemo()
- .copyIn(after, targetGroup, rule.isRewrite());
+ .copyIn(after, targetGroup, rule.isRewrite(),
+ context.getCascadesContext().getStatementContext().isDpHyp());
 
  if (result.generateNewExpression || result.correspondingExpression.getOwnerGroup() != targetGroup) {
  getEventTracer().log(TransformEvent.of(targetGroup.getLogicalExpression(), before, afters,

@@ -86,7 +86,8 @@ public final void execute() throws AnalysisException {
  }
  CopyInResult result = context.getCascadesContext()
  .getMemo()
- .copyIn(newPlan, groupExpression.getOwnerGroup(), false);
+ .copyIn(newPlan, groupExpression.getOwnerGroup(), false,
+ context.getCascadesContext().getStatementContext().isDpHyp());
  if (!result.generateNewExpression) {
  continue;
  }

@@ -245,7 +245,7 @@ public boolean simplifyGraph(int limit) {
  int upperBound = 1;
 
  // Try to probe the largest number of steps to satisfy the limit
- Counter counter = new Counter(limit);
+ Counter counter = new Counter(graph, limit);
  SubgraphEnumerator enumerator = new SubgraphEnumerator(counter, graph);
  while (true) {
  boolean hitUpperLimit = false;

@@ -145,6 +145,20 @@ public boolean enumerate() {
  return false;
  }
  }
+ // A successful enumeration must actually build the full join (the root
+ // bitmap). DPHyp can walk every CSG/CMP without any receiver returning
+ // FAIL while never inserting the root — e.g. when the only full split
+ // is rejected by the alias-dependency rule (a producer alias whose
+ // source spans both children, as in a producer-after-consumer alias
+ // chain: x on {A,B} consumed by y on {A,B,C} at split {A,C}--{B}).
+ // Accepting such a rootless probe makes GraphSimplifier stop
+ // simplifying and makes the final PlanReceiver pass return a null best
+ // plan for the root. Requiring the root bitmap forces the simplifier
+ // to keep applying steps, or the caller to fall back to the original
+ // group, instead of returning a null plan.
+ if (!receiver.contain(hyperGraph.getNodesMap())) {
+ return false;
+ }
  if (enableTrace) {
  LOG.info(traceBuilder.toString());
  }

@@ -18,10 +18,13 @@
 package org.apache.doris.nereids.jobs.joinorder.hypergraphv2.receiver;
 
 import org.apache.doris.common.Pair;
+import org.apache.doris.nereids.jobs.joinorder.hypergraphv2.HyperGraph;
 import org.apache.doris.nereids.jobs.joinorder.hypergraphv2.bitmap.LongBitmap;
 import org.apache.doris.nereids.jobs.joinorder.hypergraphv2.edge.Edge;
 import org.apache.doris.nereids.memo.Group;
 
+import java.util.BitSet;
+import java.util.HashMap;
 import java.util.List;
 
 /**
@@ -39,6 +42,56 @@ public abstract class AbstractReceiver {
 
  public abstract Group getBestPlan(long bitSet);
 
+ /**
+ * Find all edges that are missed by the current connection edges but whose
+ * reference nodes are a subset of the joined nodes. These missed edges
+ * must be added to the emitted join (they become additional join
+ * conditions). If any missed edge is enforced-order, or references a
+ * projected alias whose source spans both children, the csg-cmp pair is
+ * rejected (returns false).
+ *
+ * <p>This logic is shared by {@link PlanReceiver} (which actually emits
+ * the plan) and {@link Counter} (which counts csg-cmp pairs during graph
+ * simplification). Keeping them consistent is essential: GraphSimplifier
+ * relies on Counter to decide how many simplification steps are needed to
+ * satisfy {@code dphyperLimit}, and an over-count would over-constrain the
+ * graph and change the enumeration output.
+ */
+ protected boolean processMissedEdges(HyperGraph hyperGraph, HashMap<Long, BitSet> usdEdges,
+ long left, long right, List<Edge> edges, List<Edge> missingEdges) {
+ // find all used edges
+ BitSet usedEdgesBitmap = new BitSet();
+ usedEdgesBitmap.or(usdEdges.get(left));
+ usedEdgesBitmap.or(usdEdges.get(right));
+ edges.forEach(edge -> usedEdgesBitmap.set(edge.getIndex()));
+
+ // find all referenced nodes
+ long allReferenceNodes = LongBitmap.or(left, right);
+
+ // find the edge which is not in usedEdgesBitmap and its referenced nodes is subset of allReferenceNodes
+ for (Edge edge : hyperGraph.getJoinEdges()) {
+ if (LongBitmap.isSubset(edge.getReferenceNodes(), allReferenceNodes)
+ && !usedEdgesBitmap.get(edge.getIndex())) {
+ if (edge.isEnforcedOrder()) {
+ return false;
+ } else {
+ // Reject missed edges that reference a projected alias whose
+ // source bitmap spans both children. The alias layer is emitted
+ // by proposeProject (after proposeJoin), so the join predicate
+ // would reference a slot that does not exist in either child's
+ // output. Wait for a later join step where the alias source is
+ // fully contained in one child.
+ if (!hyperGraph.isEdgeSafeForJoin(edge, left, right)) {
+ return false;
+ }
+ // add the missed edge to edges
+ missingEdges.add(edge);
+ }
+ }
+ }
+ return true;
+ }
+
  /**
  * checkConflictRule for CD-C
  */

@@ -17,12 +17,15 @@
 
 package org.apache.doris.nereids.jobs.joinorder.hypergraphv2.receiver;
 
+import org.apache.doris.nereids.jobs.joinorder.hypergraphv2.HyperGraph;
 import org.apache.doris.nereids.jobs.joinorder.hypergraphv2.bitmap.LongBitmap;
 import org.apache.doris.nereids.jobs.joinorder.hypergraphv2.edge.Edge;
 import org.apache.doris.nereids.memo.Group;
 
 import com.google.common.base.Preconditions;
 
+import java.util.ArrayList;
+import java.util.BitSet;
 import java.util.HashMap;
 import java.util.List;
 
@@ -32,14 +35,18 @@
 public class Counter extends AbstractReceiver {
  // limit define the max number of csg-cmp pair in this Receiver
  private final int limit;
+ private final HyperGraph hyperGraph;
  private final HashMap<Long, Integer> counter = new HashMap<>();
+ private final HashMap<Long, BitSet> usdEdges = new HashMap<>();
  private int emitCount = 0;
 
- public Counter() {
+ public Counter(HyperGraph hyperGraph) {
+ this.hyperGraph = hyperGraph;
  this.limit = Integer.MAX_VALUE;
  }
 
- public Counter(int limit) {
+ public Counter(HyperGraph hyperGraph, int limit) {
+ this.hyperGraph = hyperGraph;
  this.limit = limit;
  }
 
@@ -54,13 +61,34 @@ public Counter(int limit) {
  public EmitState emitCsgCmp(long left, long right, List<Edge> edges) {
  Preconditions.checkArgument(counter.containsKey(left));
  Preconditions.checkArgument(counter.containsKey(right));
- if (!checkConflictRule(left, right, edges)) {
+ // Mirror PlanReceiver.emitCsgCmp: find missed edges first, reject the
+ // pair when an enforced-order / unsafe alias edge is found, then count
+ // the pair before the conflict-rule and alias-dependency checks (same
+ // ordering as PlanReceiver, so GraphSimplifier's limit decision matches
+ // what PlanReceiver actually emits).
+ List<Edge> missingEdges = new ArrayList<>();
+ if (!processMissedEdges(hyperGraph, usdEdges, left, right, edges, missingEdges)) {
  return EmitState.CONTINUE;
  }
  emitCount += 1;
  if (emitCount > limit) {
  return EmitState.FAIL;
  }
+ edges.addAll(missingEdges);
+ if (!checkConflictRule(left, right, edges)) {
+ return EmitState.CONTINUE;
+ }
+ // Reject cross-bitmap alias layer dependencies, same as PlanReceiver.
+ if (hyperGraph.hasUnresolvableAliasDependency(left, right)) {
+ return EmitState.CONTINUE;
+ }
+ // track used edges for the joined bitmap, same as PlanReceiver
+ BitSet usedEdgesBitmap = new BitSet();
+ usedEdgesBitmap.or(usdEdges.get(left));
+ usedEdgesBitmap.or(usdEdges.get(right));
+ edges.forEach(edge -> usedEdgesBitmap.set(edge.getIndex()));
+ usdEdges.put(LongBitmap.newBitmapUnion(left, right), usedEdgesBitmap);
+
  long bitmap = LongBitmap.newBitmapUnion(left, right);
  if (!counter.containsKey(bitmap)) {
  counter.put(bitmap, counter.get(left) * counter.get(right));
@@ -72,6 +100,7 @@ public EmitState emitCsgCmp(long left, long right, List<Edge> edges) {
 
  public void addGroup(long bitmap, Group group) {
  counter.put(bitmap, 1);
+ usdEdges.put(bitmap, new BitSet());
  }
 
  public boolean contain(long bitmap) {
@@ -80,6 +109,7 @@ public boolean contain(long bitmap) {
 
  public void reset() {
  this.counter.clear();
+ this.usdEdges.clear();
  emitCount = 0;
  }
 

@@ -160,6 +160,18 @@ public EmitState emitCsgCmp(long left, long right, List<Edge> edges) {
  LogicalPlan logicalJoin = proposeJoin(joinType, leftPlan, rightPlan, hashConjuncts,
  otherConjuncts);
 
+ // Reject join orders where cross-bitmap alias layers have an
+ // unresolvable dependency — a later layer references an alias from
+ // an earlier layer whose source spans both children. The producer
+ // layer must be fully contained in one child before the consumer
+ // can be emitted, otherwise CheckAfterRewrite rejects the plan.
+ if (hyperGraph.hasUnresolvableAliasDependency(left, right)) {
+ if (fullKeyEmitted) {
+ missingEdgeFail = true;
+ }
+ return EmitState.CONTINUE;
+ }
+
  LogicalPlan logicalPlan = proposeProject(logicalJoin, edges, left, right);
 
  // Second, we copy all physical plan to Group and generate properties and calculate cost
@@ -211,29 +223,10 @@ private Set<Slot> calculateRequiredSlots(long left, long right, List<Edge> edges
  // The root cause is hyper predicate should be encoded as one or more hyper edges in different scenarios.
  // But we are not able to do so in all cases (complex expression and outer joins).
  // So we use processMissedEdges to find all valid edges when join 0, 1, 2 as fallback plan.
+ // The logic is shared with Counter (see AbstractReceiver.processMissedEdges) so that
+ // GraphSimplifier's pair count matches what PlanReceiver actually emits.
  private boolean processMissedEdges(long left, long right, List<Edge> edges, List<Edge> missingEdges) {
- // find all used edges
- BitSet usedEdgesBitmap = new BitSet();
- usedEdgesBitmap.or(usdEdges.get(left));
- usedEdgesBitmap.or(usdEdges.get(right));
- edges.forEach(edge -> usedEdgesBitmap.set(edge.getIndex()));
-
- // find all referenced nodes
- long allReferenceNodes = LongBitmap.or(left, right);
-
- // find the edge which is not in usedEdgesBitmap and its referenced nodes is subset of allReferenceNodes
- for (Edge edge : hyperGraph.getJoinEdges()) {
- if (LongBitmap.isSubset(edge.getReferenceNodes(), allReferenceNodes)
- && !usedEdgesBitmap.get(edge.getIndex())) {
- if (edge.isEnforcedOrder()) {
- return false;
- } else {
- // add the missed edge to edges
- missingEdges.add(edge);
- }
- }
- }
- return true;
+ return super.processMissedEdges(hyperGraph, usdEdges, left, right, edges, missingEdges);
  }
 
  private void proposeAllDistributedPlans(GroupExpression groupExpression) {
@@ -291,29 +284,86 @@ public Group getBestPlan(long bitmap) {
 
  private LogicalPlan proposeProject(LogicalPlan join, List<Edge> edges, long left, long right) {
  Set<Slot> outputSet = join.getOutputSet();
- // calculate required columns by all parents
- Set<Slot> requireSlots = calculateRequiredSlots(left, right, edges);
+ // calculate required columns by all parents (final outputs + unused edges)
+ Set<Slot> parentRequireSlots = calculateRequiredSlots(left, right, edges);
+ // Pending projected aliases may reference input slots (e.g., A.v, B.v for
+ // s=A.v+B.v) that are not in finalRequiredSlots or unused edges. Preserve
+ // them so the join output still contains the base columns needed to evaluate
+ // the alias expressions, both for aliases emitted at this stage and those
+ // deferred to a later join whose bitmap is a superset.
+ Set<Slot> aliasInputSlots = hyperGraph.getAllAliasInputSlotsForNodes(
+ LongBitmap.newBitmapUnion(left, right));
+ Set<Slot> requireSlots = new HashSet<>(parentRequireSlots);
+ requireSlots.addAll(aliasInputSlots);
  List<NamedExpression> allProjects = new ArrayList<>(outputSet.size());
  for (Slot slot : outputSet) {
  if (requireSlots.contains(slot)) {
  allProjects.add(slot);
  }
  }
- if (hyperGraph.hasLiteralAlias()) {
- allProjects.addAll(hyperGraph.getLiteralAlias(left, right));
- }
 
  if (allProjects.isEmpty()) {
  allProjects.add(new Alias(new ExprId(-1), new TinyIntLiteral((byte) 1)));
  }
 
- // propose logical project
+ // propose logical project for the slot pass-through
  LogicalPlan logicalPlan;
  if (outputSet.equals(new HashSet<>(allProjects))) {
  logicalPlan = join;
  } else {
  logicalPlan = new LogicalProject<>(allProjects, join);
  }
+
+ // Emit projected aliases as a single LogicalProject node.
+ // Cross-layer references (e.g., z = x + 1 referencing x = COALESCE(v, 0))
+ // were already resolved at graph-build time, so only one Project is needed.
+ // Carry forward child slots still required by parents (e.g., join keys)
+ // or by deferred alias layers (e.g., B.w for a later y=B.w+1).
+ // Use the full requireSlots so that deferred-layer inputs survive
+ // through intermediate layers.
+ if (hyperGraph.hasProjectedAliases()) {
+ List<NamedExpression> aliases = hyperGraph.getProjectedAliases(left, right);
+ if (!aliases.isEmpty()) {
+ Set<ExprId> aliasExprIds = new HashSet<>();
+ for (NamedExpression a : aliases) {
+ aliasExprIds.add(a.getExprId());
+ }
+ List<NamedExpression> mergedLayer = new ArrayList<>(aliases);
+ // Decide which raw base columns can be dropped from this alias
+ // layer's carry-forward. A base column is an alias raw input
+ // that is consumed once every alias reading it has been
+ // materialized within this union. It must be KEPT when:
+ // - it is still required by the final output, or
+ // - a deferred alias (source NOT inside this union) reads it
+ // (that alias is materialized at a later join step and
+ // reads the raw base column from the join output), or
+ // - some join edge references it (e.g. a pushed-down hash
+ // expression such as `col + 1 = key` reads the raw base
+ // column from the join output).
+ // Every exclusion above is computed from the graph + union
+ // only, never from the left/right split, so all decompositions
+ // of the same bitmap produce the same plan output set (memo
+ // output-set consistency). A split-dependent drop would let
+ // one ordering drop a column that another ordering still needs,
+ // producing "Input slot(s) not in child's output" failures.
+ Set<Slot> exclusivelyEmittedSlots = new HashSet<>(aliasInputSlots);
+ exclusivelyEmittedSlots.removeAll(finalRequiredSlots);
+ exclusivelyEmittedSlots.removeAll(hyperGraph.getDeferredAliasInputSlotsForNodes(
+ LongBitmap.newBitmapUnion(left, right)));
+ for (Edge edge : hyperGraph.getJoinEdges()) {
+ exclusivelyEmittedSlots.removeAll(edge.getInputSlots());
+ }
+ for (Slot childSlot : logicalPlan.getOutputSet()) {
+ if (requireSlots.contains(childSlot)
+ && !aliasExprIds.contains(childSlot.getExprId())
+ && !exclusivelyEmittedSlots.contains(childSlot)) {
+ mergedLayer.add(childSlot);
+ }
+ }
+ logicalPlan = new LogicalProject<>(mergedLayer, logicalPlan);
+ }
+ }
+
  if (LongBitmap.newBitmapUnion(left, right) == allNodeBitmap
  && !logicalPlan.getOutputSet().equals(new HashSet<>(finalProjects))) {
  logicalPlan = new LogicalProject<>(finalProjects, logicalPlan);

@@ -292,11 +292,29 @@ public CopyInResult copyIn(Plan plan, @Nullable Group target, HashMap<Long, Grou
  * is the corresponding group expression of the plan
  */
  public CopyInResult copyIn(Plan plan, @Nullable Group target, boolean rewrite) {
+ return copyIn(plan, target, rewrite, false);
+ }
+
+ /**
+ * Add plan to Memo.
+ *
+ * @param plan {@link Plan} or {@link Expression} to be added
+ * @param target target group to add node. null to generate new Group
+ * @param rewrite whether to rewrite the node to the target group
+ * @param isInDpHyper whether this copy happens during DPHyp enumeration. During DPHyp
+ * the output of a group is only guaranteed to be consistent by its
+ * output slot set (nullability may legitimately differ across join
+ * orders of outer joins), so the relaxed set-only comparison is used.
+ * @return CopyInResult, in which the generateNewExpression is true if a newly generated
+ * groupExpression added into memo, and the correspondingExpression
+ * is the corresponding group expression of the plan
+ */
+ public CopyInResult copyIn(Plan plan, @Nullable Group target, boolean rewrite, boolean isInDpHyper) {
  CopyInResult result;
  if (rewrite) {
  result = doRewrite(plan, target);
  } else {
- result = doCopyIn(plan, target, null, false);
+ result = doCopyIn(plan, target, null, isInDpHyper);
  }
  maybeAddStateId(result);
  return result;