nprasad2077/doris
1
0 Fork 0
mirror of https://github.com/apache/doris.git synced 2026-11-04 07:31:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

[fix](short-circuit) flatten multi-layer projections in short-circuit point queries (#61177)

Browse Source 
When Nereids generates multi-layer projections (e.g., for expressions
like `IFNULL(a, 0) + IFNULL(b, 0)`), the intermediate projection layers
reference slots from intermediate tuples. In short-circuit (point query)
execution, ShortCircuitQueryContext serializes the projection list and
sends it to BE. However, BE evaluates projections directly on scan
output and has no knowledge of intermediate tuples, causing incorrect
results or crashes when the projection references intermediate slot IDs.

This fix adds PlanNode.getPointQueryProjectList() which flattens all
intermediate projection layers into a single-layer projection by
substituting intermediate SlotRefs with their source expressions via
ExprSubstitutionMap. ShortCircuitQueryContext now calls
getPointQueryProjectList() instead of getProjectList() to ensure BE
receives fully resolved expressions.
This commit is contained in:
lihangyu authored and GitHub committed 2026-03-11 21:44:05 +08:00
1 parent 07cff332a3
commit 44d721d2fe
5 files changed
+495 -2
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

fe/fe-core/src/main/java/org/apache/doris/planner/PlanNode.java
+30
View File
@@ -22,6 +22,7 @@ package org.apache.doris.planner;
import org.apache.doris.analysis.CompoundPredicate;
import org.apache.doris.analysis.Expr;
import org.apache.doris.analysis.ExprSubstitutionMap;
import org.apache.doris.analysis.ExprToSqlVisitor;
import org.apache.doris.analysis.ExprToThriftVisitor;
import org.apache.doris.analysis.SlotDescriptor;
@@ -743,6 +744,21 @@ public abstract class PlanNode extends TreeNode<PlanNode> {
return projectList;
}
public List<Expr> getPointQueryProjectList() {
if (CollectionUtils.isEmpty(projectList) || intermediateProjectListList.isEmpty()) {
return projectList;
}
List<Expr> flattenedProjectList = Expr.cloneList(projectList);
for (int i = intermediateProjectListList.size() - 1; i >= 0; --i) {
flattenedProjectList = Expr.cloneList(
flattenedProjectList,
createPointQueryProjectionSmap(intermediateOutputTupleDescList.get(i),
intermediateProjectListList.get(i)));
}
return flattenedProjectList;
}
public void setCardinalityAfterFilter(long cardinalityAfterFilter) {
this.cardinalityAfterFilter = cardinalityAfterFilter;
}
@@ -773,6 +789,20 @@ public abstract class PlanNode extends TreeNode<PlanNode> {
intermediateProjectListList.add(exprs);
}
private ExprSubstitutionMap createPointQueryProjectionSmap(
TupleDescriptor outputTupleDesc, List<Expr> projectionExprs) {
List<SlotDescriptor> outputSlots = outputTupleDesc.getSlots();
Preconditions.checkState(outputSlots.size() == projectionExprs.size(),
"point query projection slot size %s does not match expr size %s",
outputSlots.size(), projectionExprs.size());
ExprSubstitutionMap substitutionMap = new ExprSubstitutionMap();
for (int i = 0; i < outputSlots.size(); ++i) {
substitutionMap.put(new SlotRef(outputSlots.get(i)), projectionExprs.get(i));
}
return substitutionMap;
}
public <T extends PlanNode> List<T> collectInCurrentFragment(Predicate<PlanNode> predicate) {
List<PlanNode> result = Lists.newArrayList();
foreachDownInCurrentFragment(child -> {
fe/fe-core/src/main/java/org/apache/doris/qe/ShortCircuitQueryContext.java
+4 -2
View File
@@ -17,6 +17,7 @@
package org.apache.doris.qe;
import org.apache.doris.analysis.Expr;
import org.apache.doris.analysis.ExprToThriftVisitor;
import org.apache.doris.analysis.Queriable;
import org.apache.doris.catalog.OlapTable;
@@ -88,9 +89,10 @@ public class ShortCircuitQueryContext {
new TSerializer().serialize(options));
List<TExpr> exprs = new ArrayList<>();
OlapScanNode olapScanNode = (OlapScanNode) planner.getScanNodes().get(0);
if (olapScanNode.getProjectList() != null) {
List<Expr> pointQueryProjectList = olapScanNode.getPointQueryProjectList();
if (pointQueryProjectList != null) {
// project on scan node
exprs.addAll(olapScanNode.getProjectList().stream()
exprs.addAll(pointQueryProjectList.stream()
.map(ExprToThriftVisitor::treeToThrift).collect(Collectors.toList()));
} else {
// add output slots
fe/fe-core/src/test/java/org/apache/doris/planner/PlanNodeTest.java
+102
View File
@@ -0,0 +1,102 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.doris.planner;
import org.apache.doris.analysis.ArithmeticExpr;
import org.apache.doris.analysis.Expr;
import org.apache.doris.analysis.IntLiteral;
import org.apache.doris.analysis.SlotDescriptor;
import org.apache.doris.analysis.SlotId;
import org.apache.doris.analysis.SlotRef;
import org.apache.doris.analysis.TupleDescriptor;
import org.apache.doris.analysis.TupleId;
import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.Function.NullableMode;
import org.apache.doris.catalog.Type;
import org.apache.doris.thrift.TPlanNode;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
public class PlanNodeTest {
@Test
public void testGetPointQueryProjectListFlattensIntermediateProjection() {
TestPlanNode planNode = new TestPlanNode();
TupleDescriptor scanTuple = new TupleDescriptor(new TupleId(0));
SlotDescriptor scanSlotA = createColumnSlot(scanTuple, 0, "a");
SlotDescriptor scanSlotB = createColumnSlot(scanTuple, 1, "b");
TupleDescriptor intermediateTuple = new TupleDescriptor(new TupleId(1));
SlotDescriptor projectedSlotA = createColumnSlot(intermediateTuple, 2, "a");
SlotDescriptor projectedExprSlot = createExprSlot(intermediateTuple, 3, "expr_col");
planNode.addIntermediateOutputTupleDescList(intermediateTuple);
planNode.addIntermediateProjectList(Lists.newArrayList(
new SlotRef(scanSlotA),
new ArithmeticExpr(ArithmeticExpr.Operator.ADD, new SlotRef(scanSlotB),
new IntLiteral(1), Type.BIGINT, NullableMode.DEPEND_ON_ARGUMENT, true)));
planNode.setProjectList(Lists.newArrayList(
new ArithmeticExpr(ArithmeticExpr.Operator.ADD, new SlotRef(projectedSlotA),
new SlotRef(projectedExprSlot), Type.BIGINT,
NullableMode.DEPEND_ON_ARGUMENT, true)));
List<Expr> pointQueryProjectList = planNode.getPointQueryProjectList();
Assertions.assertEquals(1, pointQueryProjectList.size());
Set<SlotId> slotIds = new HashSet<>();
Expr.extractSlots(pointQueryProjectList.get(0), slotIds);
Assertions.assertEquals(Sets.newHashSet(scanSlotA.getId(), scanSlotB.getId()), slotIds);
Assertions.assertFalse(slotIds.contains(projectedSlotA.getId()));
Assertions.assertFalse(slotIds.contains(projectedExprSlot.getId()));
}
private static SlotDescriptor createColumnSlot(TupleDescriptor tupleDescriptor, int slotId,
String columnName) {
SlotDescriptor slotDescriptor = new SlotDescriptor(new SlotId(slotId), tupleDescriptor);
slotDescriptor.setColumn(new Column(columnName, Type.BIGINT));
slotDescriptor.setLabel(columnName);
tupleDescriptor.addSlot(slotDescriptor);
return slotDescriptor;
}
private static SlotDescriptor createExprSlot(TupleDescriptor tupleDescriptor, int slotId,
String label) {
SlotDescriptor slotDescriptor = new SlotDescriptor(new SlotId(slotId), tupleDescriptor);
slotDescriptor.setType(Type.BIGINT);
slotDescriptor.setLabel(label);
tupleDescriptor.addSlot(slotDescriptor);
return slotDescriptor;
}
private static class TestPlanNode extends PlanNode {
TestPlanNode() {
super(new PlanNodeId(0), "TEST");
}
@Override
protected void toThrift(TPlanNode msg) {
}
}
}
regression-test/data/point_query_p0/test_short_circuit_project.out
+37
View File
@@ -0,0 +1,37 @@
-- This file is automatically generated. You should know what you did if you want to edit this
-- !sql --
7 9.5
-- !arith --
410 4 11.57
-- !cast --
10 hello 3 10.5
-- !case_when --
big 6.28
-- !string_fn --
HELLO WORLD! 11 hello
-- !math_fn --
10 4 3 3.1 100.0
-- !null_fn --
10 \N hello world 3.14
-- !if_cmp --
200 200 1.0
-- !nested --
210-pos 313.97
-- !simple --
10 hello world
-- !null_row --
0 empty null_int -1
-- !neg_arith --
5 25 -5.0
regression-test/suites/point_query_p0/test_short_circuit_project.groovy
+322
View File
@@ -0,0 +1,322 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
suite("test_short_circuit_project", "p0") {
// ---------- common table setup ----------
def tableName = "cir_19608_short_circuit_project"
sql "DROP TABLE IF EXISTS ${tableName}"
sql """
CREATE TABLE ${tableName} (
`uuid` BIGINT NULL,
`interstitial_ad_watch_cnt_lt` BIGINT NULL,
`splash_ad_watch_cnt_lt` BIGINT NULL,
`splash_ad_revenue_lt` DOUBLE NULL
) ENGINE=OLAP
UNIQUE KEY(`uuid`)
DISTRIBUTED BY HASH(`uuid`) BUCKETS 1
PROPERTIES (
"replication_allocation" = "tag.location.default: 1",
"store_row_column" = "true",
"enable_unique_key_merge_on_write" = "true",
"light_schema_change" = "true",
"storage_format" = "V2"
)
"""
sql """
INSERT INTO ${tableName} VALUES
(10001, 3, 4, 5.5)
"""
// ---------- original test: IFNULL + ADD ----------
explain {
sql """
SELECT /*+ SET_VAR(enable_nereids_planner=true) */
IFNULL(interstitial_ad_watch_cnt_lt, 0) + IFNULL(splash_ad_watch_cnt_lt, 0),
IFNULL(splash_ad_revenue_lt, 0) + IFNULL(splash_ad_watch_cnt_lt, 0)
FROM ${tableName}
WHERE uuid = 10001
"""
contains "SHORT-CIRCUIT"
}
qt_sql """
SELECT /*+ SET_VAR(enable_nereids_planner=true) */
IFNULL(interstitial_ad_watch_cnt_lt, 0) + IFNULL(splash_ad_watch_cnt_lt, 0),
IFNULL(splash_ad_revenue_lt, 0) + IFNULL(splash_ad_watch_cnt_lt, 0)
FROM ${tableName}
WHERE uuid = 10001
"""
// ---------- extended table with richer types ----------
def extTable = "short_circuit_project_ext"
sql "DROP TABLE IF EXISTS ${extTable}"
sql """
CREATE TABLE ${extTable} (
`id` BIGINT NOT NULL,
`col_int` INT NULL,
`col_bigint` BIGINT NULL,
`col_double` DOUBLE NULL,
`col_decimal` DECIMALV3(18, 4) NULL,
`col_str` VARCHAR(200) NULL,
`col_date` DATE NULL,
`col_datetime` DATETIME NULL
) ENGINE=OLAP
UNIQUE KEY(`id`)
DISTRIBUTED BY HASH(`id`) BUCKETS 1
PROPERTIES (
"replication_allocation" = "tag.location.default: 1",
"store_row_column" = "true",
"enable_unique_key_merge_on_write" = "true",
"light_schema_change" = "true",
"storage_format" = "V2"
)
"""
sql """
INSERT INTO ${extTable} VALUES
(1, 10, 200, 3.14, 99.9900, 'hello world', '2026-03-10', '2026-03-10 12:30:00'),
(2, -5, 0, 0.0, 0.0000, '', '2000-01-01', '2000-01-01 00:00:00'),
(3, NULL, NULL, NULL, NULL, NULL, NULL, NULL)
"""
// ---------- 1. Arithmetic operators: +, -, *, /, % ----------
explain {
sql """
SELECT /*+ SET_VAR(enable_nereids_planner=true) */
(col_int + col_bigint) * 2 - col_int,
col_bigint % 7,
col_double / 2.0 + col_int
FROM ${extTable} WHERE id = 1
"""
contains "SHORT-CIRCUIT"
}
qt_arith """
SELECT /*+ SET_VAR(enable_nereids_planner=true) */
(col_int + col_bigint) * 2 - col_int,
col_bigint % 7,
col_double / 2.0 + col_int
FROM ${extTable} WHERE id = 1
"""
// ---------- 2. CAST / type coercion ----------
explain {
sql """
SELECT /*+ SET_VAR(enable_nereids_planner=true) */
CAST(col_int AS VARCHAR(20)),
CAST(col_str AS VARCHAR(5)),
CAST(col_double AS BIGINT),
CAST(col_int AS DOUBLE) + 0.5
FROM ${extTable} WHERE id = 1
"""
contains "SHORT-CIRCUIT"
}
qt_cast """
SELECT /*+ SET_VAR(enable_nereids_planner=true) */
CAST(col_int AS VARCHAR(20)),
CAST(col_str AS VARCHAR(5)),
CAST(col_double AS BIGINT),
CAST(col_int AS DOUBLE) + 0.5
FROM ${extTable} WHERE id = 1
"""
// ---------- 3. CASE WHEN ----------
explain {
sql """
SELECT /*+ SET_VAR(enable_nereids_planner=true) */
CASE WHEN col_int > 5 THEN 'big' WHEN col_int > 0 THEN 'small' ELSE 'zero_or_neg' END,
CASE WHEN col_double > 3.0 THEN col_double * 2 ELSE col_double END
FROM ${extTable} WHERE id = 1
"""
contains "SHORT-CIRCUIT"
}
qt_case_when """
SELECT /*+ SET_VAR(enable_nereids_planner=true) */
CASE WHEN col_int > 5 THEN 'big' WHEN col_int > 0 THEN 'small' ELSE 'zero_or_neg' END,
CASE WHEN col_double > 3.0 THEN col_double * 2 ELSE col_double END
FROM ${extTable} WHERE id = 1
"""
// ---------- 4. String functions ----------
explain {
sql """
SELECT /*+ SET_VAR(enable_nereids_planner=true) */
CONCAT(UPPER(col_str), '!'),
LENGTH(col_str),
SUBSTR(col_str, 1, 5)
FROM ${extTable} WHERE id = 1
"""
contains "SHORT-CIRCUIT"
}
qt_string_fn """
SELECT /*+ SET_VAR(enable_nereids_planner=true) */
CONCAT(UPPER(col_str), '!'),
LENGTH(col_str),
SUBSTR(col_str, 1, 5)
FROM ${extTable} WHERE id = 1
"""
// ---------- 5. Math functions ----------
explain {
sql """
SELECT /*+ SET_VAR(enable_nereids_planner=true) */
ABS(col_int - 20),
CEIL(col_double),
FLOOR(col_double),
ROUND(col_double, 1),
POWER(col_int, 2)
FROM ${extTable} WHERE id = 1
"""
contains "SHORT-CIRCUIT"
}
qt_math_fn """
SELECT /*+ SET_VAR(enable_nereids_planner=true) */
ABS(col_int - 20),
CEIL(col_double),
FLOOR(col_double),
ROUND(col_double, 1),
POWER(col_int, 2)
FROM ${extTable} WHERE id = 1
"""
// ---------- 6. NULL handling: COALESCE, NULLIF ----------
explain {
sql """
SELECT /*+ SET_VAR(enable_nereids_planner=true) */
COALESCE(col_int, col_bigint, -1),
NULLIF(col_int, 10),
IFNULL(col_str, 'N/A'),
COALESCE(NULL, col_double, 0)
FROM ${extTable} WHERE id = 1
"""
contains "SHORT-CIRCUIT"
}
qt_null_fn """
SELECT /*+ SET_VAR(enable_nereids_planner=true) */
COALESCE(col_int, col_bigint, -1),
NULLIF(col_int, 10),
IFNULL(col_str, 'N/A'),
COALESCE(NULL, col_double, 0)
FROM ${extTable} WHERE id = 1
"""
// ---------- 7. IF / comparison: IF, GREATEST, LEAST ----------
explain {
sql """
SELECT /*+ SET_VAR(enable_nereids_planner=true) */
IF(col_int > col_bigint, col_int, col_bigint),
GREATEST(col_int, col_bigint, 100),
LEAST(col_double, CAST(col_int AS DOUBLE), 1.0)
FROM ${extTable} WHERE id = 1
"""
contains "SHORT-CIRCUIT"
}
qt_if_cmp """
SELECT /*+ SET_VAR(enable_nereids_planner=true) */
IF(col_int > col_bigint, col_int, col_bigint),
GREATEST(col_int, col_bigint, 100),
LEAST(col_double, CAST(col_int AS DOUBLE), 1.0)
FROM ${extTable} WHERE id = 1
"""
// ---------- 8. Deeply nested multi-operator expressions ----------
explain {
sql """
SELECT /*+ SET_VAR(enable_nereids_planner=true) */
CAST(IF(col_int > 0,
CONCAT(CAST(col_int + col_bigint AS VARCHAR(50)), '-pos'),
CONCAT(CAST(ABS(col_int) AS VARCHAR(50)), '-neg')) AS VARCHAR(100)),
ROUND(COALESCE(col_double, 0) * IFNULL(col_decimal, 1.0), 2)
FROM ${extTable} WHERE id = 1
"""
contains "SHORT-CIRCUIT"
}
qt_nested """
SELECT /*+ SET_VAR(enable_nereids_planner=true) */
CAST(IF(col_int > 0,
CONCAT(CAST(col_int + col_bigint AS VARCHAR(50)), '-pos'),
CONCAT(CAST(ABS(col_int) AS VARCHAR(50)), '-neg')) AS VARCHAR(100)),
ROUND(COALESCE(col_double, 0) * IFNULL(col_decimal, 1.0), 2)
FROM ${extTable} WHERE id = 1
"""
// ---------- 9. Simple projection (no intermediate layer) ----------
explain {
sql """
SELECT /*+ SET_VAR(enable_nereids_planner=true) */
col_int, col_str
FROM ${extTable} WHERE id = 1
"""
contains "SHORT-CIRCUIT"
}
qt_simple """
SELECT /*+ SET_VAR(enable_nereids_planner=true) */
col_int, col_str
FROM ${extTable} WHERE id = 1
"""
// ---------- 10. NULL row with complex expressions ----------
explain {
sql """
SELECT /*+ SET_VAR(enable_nereids_planner=true) */
IFNULL(col_int, 0) + IFNULL(col_bigint, 0),
COALESCE(col_str, 'empty'),
CASE WHEN col_int IS NULL THEN 'null_int' ELSE 'has_int' END,
CAST(IFNULL(col_double, -1) AS BIGINT)
FROM ${extTable} WHERE id = 3
"""
contains "SHORT-CIRCUIT"
}
qt_null_row """
SELECT /*+ SET_VAR(enable_nereids_planner=true) */
IFNULL(col_int, 0) + IFNULL(col_bigint, 0),
COALESCE(col_str, 'empty'),
CASE WHEN col_int IS NULL THEN 'null_int' ELSE 'has_int' END,
CAST(IFNULL(col_double, -1) AS BIGINT)
FROM ${extTable} WHERE id = 3
"""
// ---------- 11. Negative values with arithmetic ----------
explain {
sql """
SELECT /*+ SET_VAR(enable_nereids_planner=true) */
ABS(col_int) + ABS(col_bigint),
col_int * col_int,
IFNULL(col_double, 0) + IFNULL(CAST(col_int AS DOUBLE), 0)
FROM ${extTable} WHERE id = 2
"""
contains "SHORT-CIRCUIT"
}
qt_neg_arith """
SELECT /*+ SET_VAR(enable_nereids_planner=true) */
ABS(col_int) + ABS(col_bigint),
col_int * col_int,
IFNULL(col_double, 0) + IFNULL(CAST(col_int AS DOUBLE), 0)
FROM ${extTable} WHERE id = 2
"""
}