Skip to content

[CALCITE-5845] Select a very large predicate throw threw 'java.lang.StackOverflowError' exception in SqlNode to RelNode phase #5384

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
xuzifu666 merged 1 commit into from
Nov 5, 2026
Merged

[CALCITE-5845] Select a very large predicate throw threw 'java.lang.StackOverflowError' exception in SqlNode to RelNode phase #5384

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
126 changes: 81 additions & 45 deletions core/src/main/java/org/apache/calcite/plan/RelOptUtil.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2559,15 +2559,27 @@ public static List<RelDataTypeField> deduplicateColumns(
public static void decomposeConjunction(
@Nullable RexNode rexPredicate,
List<RexNode> rexList) {
if (rexPredicate == null || rexPredicate.isAlwaysTrue()) {
// Use an explicit stack to avoid StackOverflowError on deeply nested ANDs
// (e.g. CALCITE-5845)
if (rexPredicate == null) {
return;
}
if (rexPredicate.isA(SqlKind.AND)) {
for (RexNode operand : ((RexCall) rexPredicate).getOperands()) {
decomposeConjunction(operand, rexList);
final Deque<RexNode> stack = new ArrayDeque<>();
stack.push(rexPredicate);
while (!stack.isEmpty()) {
final RexNode node = stack.pop();
if (node.isAlwaysTrue()) {
continue;
}
if (node.isA(SqlKind.AND)) {
final List<RexNode> operands = ((RexCall) node).getOperands();
// Push in reverse order so that the leftmost operand is processed first
for (int i = operands.size() - 1; i >= 0; i--) {
stack.push(operands.get(i));
}
} else {
rexList.add(node);
}
} else {
rexList.add(rexPredicate);
}
}

Expand All @@ -2593,47 +2605,59 @@ public static void decomposeConjunction(
@Nullable RexNode rexPredicate,
List<RexNode> rexList,
List<RexNode> notList) {
if (rexPredicate == null || rexPredicate.isAlwaysTrue()) {
// Use an explicit stack to avoid StackOverflowError on deeply nested
// expressions (e.g. CALCITE-5845)
if (rexPredicate == null) {
return;
}
switch (rexPredicate.getKind()) {
case AND:
for (RexNode operand : ((RexCall) rexPredicate).getOperands()) {
decomposeConjunction(operand, rexList, notList);
}
break;
case NOT:
final RexNode e = ((RexCall) rexPredicate).getOperands().get(0);
if (e.isAlwaysFalse()) {
return;
final Deque<RexNode> stack = new ArrayDeque<>();
stack.push(rexPredicate);
while (!stack.isEmpty()) {
final RexNode node = stack.pop();
if (node.isAlwaysTrue()) {
continue;
}
switch (e.getKind()) {
case OR:
final List<RexNode> ors = new ArrayList<>();
decomposeDisjunction(e, ors);
for (RexNode or : ors) {
switch (or.getKind()) {
case NOT:
rexList.add(((RexCall) or).operands.get(0));
break;
default:
notList.add(or);
switch (node.getKind()) {
case AND:
final List<RexNode> operands = ((RexCall) node).getOperands();
// Push in reverse order so that the leftmost operand is processed first
for (int i = operands.size() - 1; i >= 0; i--) {
stack.push(operands.get(i));
}
break;
case NOT:
final RexNode e = ((RexCall) node).getOperands().get(0);
if (e.isAlwaysFalse()) {
break;
}
switch (e.getKind()) {
case OR:
final List<RexNode> ors = new ArrayList<>();
decomposeDisjunction(e, ors);
for (RexNode or : ors) {
switch (or.getKind()) {
case NOT:
rexList.add(((RexCall) or).operands.get(0));
break;
default:
notList.add(or);
}
}
break;
default:
notList.add(e);
}
break;
case LITERAL:
if (!RexLiteral.isNullLiteral(node)
&& RexLiteral.booleanValue(node)) {
break; // ignore TRUE
}
// fall through
default:
notList.add(e);
}
break;
case LITERAL:
if (!RexLiteral.isNullLiteral(rexPredicate)
&& RexLiteral.booleanValue(rexPredicate)) {
return; // ignore TRUE
rexList.add(node);
break;
}
// fall through
default:
rexList.add(rexPredicate);
break;
}
}

Expand All @@ -2647,15 +2671,27 @@ public static void decomposeConjunction(
public static void decomposeDisjunction(
@Nullable RexNode rexPredicate,
List<RexNode> rexList) {
if (rexPredicate == null || rexPredicate.isAlwaysFalse()) {
// Use an explicit stack to avoid StackOverflowError on deeply nested ORs
// (e.g. CALCITE-5845)
if (rexPredicate == null) {
return;
}
if (rexPredicate.isA(SqlKind.OR)) {
for (RexNode operand : ((RexCall) rexPredicate).getOperands()) {
decomposeDisjunction(operand, rexList);
final Deque<RexNode> stack = new ArrayDeque<>();
stack.push(rexPredicate);
while (!stack.isEmpty()) {
final RexNode node = stack.pop();
if (node.isAlwaysFalse()) {
continue;
}
if (node.isA(SqlKind.OR)) {
final List<RexNode> operands = ((RexCall) node).getOperands();
// Push in reverse order so that the leftmost operand is processed first
for (int i = operands.size() - 1; i >= 0; i--) {
stack.push(operands.get(i));
}
} else {
rexList.add(node);
}
} else {
rexList.add(rexPredicate);
}
}

Expand Down
65 changes: 49 additions & 16 deletions core/src/main/java/org/apache/calcite/rex/RexUtil.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -67,10 +67,12 @@

import java.math.BigDecimal;
import java.math.RoundingMode;
import java.util.ArrayDeque;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.Deque;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
Expand Down Expand Up @@ -1519,15 +1521,22 @@ public static ImmutableList<RexNode> flattenAnd(

private static void addAnd(ImmutableList.Builder<RexNode> builder,
Set<RexNode> digests, RexNode node) {
switch (node.getKind()) {
case AND:
for (RexNode operand : ((RexCall) node).getOperands()) {
addAnd(builder, digests, operand);
}
return;
default:
if (!node.isAlwaysTrue() && digests.add(node)) {
builder.add(node);
// Use an explicit stack to avoid StackOverflowError on deeply nested ANDs
// (e.g. CALCITE-5845)
final Deque<RexNode> stack = new ArrayDeque<>();
stack.push(node);
while (!stack.isEmpty()) {
final RexNode current = stack.pop();
if (current.getKind() == SqlKind.AND) {
final List<RexNode> operands = ((RexCall) current).getOperands();
// Push in reverse order so that the leftmost operand is processed first
for (int i = operands.size() - 1; i >= 0; i--) {
stack.push(operands.get(i));
}
} else {
if (!current.isAlwaysTrue() && digests.add(current)) {
builder.add(current);
}
}
}
}
Expand Down Expand Up @@ -1823,20 +1832,44 @@ private static boolean isFlat(
* calls to an associative operator such as AND and OR.
*/
public static boolean isFlat(RexNode expr) {
if (!(expr instanceof RexCall)) {
return true;
// Use an explicit stack to avoid StackOverflowError on deeply nested
// expressions (e.g. CALCITE-5845)
final Deque<RexNode> stack = new ArrayDeque<>();
stack.push(expr);
while (!stack.isEmpty()) {
final RexNode current = stack.pop();
if (!(current instanceof RexCall)) {
continue;
}
final RexCall call = (RexCall) current;
if (!isFlat(call.getOperands(), call.getOperator())) {

@mihaibudiu mihaibudiu Nov 2, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I haven't checked if this function may be recursive.

@xuzifu666 xuzifu666 Nov 3, 2026

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It's non-recursive and it only loops over an explicit ArrayDeque, and the private list overload is the original non-recursive exists() check. testIsFlatDeeplyNested verifies it: overflows against the old code on a 256KB stack, passes in latest commit.

return false;
}
for (RexNode operand : call.getOperands()) {
stack.push(operand);
}
}
final RexCall call = (RexCall) expr;
return isFlat(call.getOperands(), call.getOperator())
&& all(call.getOperands(), RexUtil::isFlat);
return true;
}

private static void flattenRecurse(
List<RexNode> list, List<? extends RexNode> exprs, SqlOperator op) {
for (RexNode expr : exprs) {
// Use an explicit stack to avoid StackOverflowError on deeply nested
// associative calls (e.g. CALCITE-5845)
final Deque<RexNode> stack = new ArrayDeque<>();
// Push in reverse order so that the leftmost operand is processed first
for (int i = exprs.size() - 1; i >= 0; i--) {
stack.push(exprs.get(i));
}
while (!stack.isEmpty()) {
final RexNode expr = stack.pop();
if (expr instanceof RexCall
&& ((RexCall) expr).getOperator() == op) {
flattenRecurse(list, ((RexCall) expr).getOperands(), op);
final List<RexNode> operands = ((RexCall) expr).getOperands();
// Push in reverse order so that the leftmost operand is processed first
for (int i = operands.size() - 1; i >= 0; i--) {
stack.push(operands.get(i));
}
} else {
list.add(expr);
}
Expand Down
75 changes: 75 additions & 0 deletions core/src/main/java/org/apache/calcite/sql2rel/StandardConvertletTable.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -93,6 +93,7 @@
import java.math.RoundingMode;
import java.util.ArrayDeque;
import java.util.ArrayList;
import java.util.Deque;
import java.util.List;
import java.util.Objects;
import java.util.function.Function;
Expand Down Expand Up @@ -1204,6 +1205,17 @@ public RexNode convertCall(
SqlCall call) {
final SqlOperator op = call.getOperator();
final RexBuilder rexBuilder = cx.getRexBuilder();

// For associative operators (AND, OR), flatten the SqlNode tree iteratively
// to avoid StackOverflowError on deeply nested expressions.
// The parser creates a left-deep binary tree for chains like
// "a AND b AND c AND d ...", and the default convertOperands would recurse
// into each nested level, causing stack overflow for large operand counts.
final SqlKind kind = op.getKind();
if (kind == SqlKind.AND || kind == SqlKind.OR) {
return convertAssociativeCall(cx, rexBuilder, call, op, kind);
}

SqlOperandTypeChecker operandTypeChecker = op.getOperandTypeChecker();
final SqlOperandTypeChecker.Consistency consistency =
operandTypeChecker == null
Expand Down Expand Up @@ -1276,6 +1288,69 @@ public RexNode convertCall(
return rexBuilder.makeCall(call.getParserPosition(), type, op, RexUtil.flatten(exprs, op));
}

/**
* Converts a call to an associative operator (AND, OR) by iteratively
* flattening the nested binary SqlNode tree into a flat list of operands,
* then converting each operand. This avoids StackOverflowError on deeply
* nested expressions.
*/
private static RexNode convertAssociativeCall(
SqlRexContext cx,
RexBuilder rexBuilder,
SqlCall call,
SqlOperator op,
SqlKind kind) {
// Step 1: Iteratively collect all leaf operands from the nested
// binary tree. For example, AND(AND(AND(a, b), c), d) becomes [a, b, c, d].
final List<SqlNode> operands = new ArrayList<>();
final Deque<SqlNode> stack = new ArrayDeque<>();
stack.push(call);
while (!stack.isEmpty()) {
final SqlNode node = stack.pop();
if (node instanceof SqlCall && node.getKind() == kind) {
final SqlCall subCall = (SqlCall) node;
// Push right operand first, then left, so that left is processed first
final List<SqlNode> subOperands = subCall.getOperandList();
for (int i = subOperands.size() - 1; i >= 0; i--) {
stack.push(subOperands.get(i));
}
} else {
operands.add(node);
}
}

// Step 2: Convert each leaf operand.
final List<RexNode> exprs = new ArrayList<>();
for (SqlNode operand : operands) {
exprs.add(cx.convertExpression(operand));
}

// Step 3: Apply type coercion consistent with the original binary call.
// The validator has already checked the original binary call's operand types.
// We replicate the same coercion for all operands.
final List<RelDataType> operandTypes =
cx.getValidator().getValidatedOperandTypes(call);
if (operandTypes != null && !operandTypes.isEmpty()) {
// Use the first operand's validated type as the template
// (both operands of AND/OR have the same validated type)
final RelDataType targetType = operandTypes.get(0);
final List<RexNode> coercedExprs = new ArrayList<>();
for (RexNode expr : exprs) {
coercedExprs.add(
rexBuilder.ensureType(call.getParserPosition(), targetType, expr, true));
}
exprs.clear();
exprs.addAll(coercedExprs);
}

// Step 4: Flatten any nested AND/OR RexNodes that may have resulted from
// operand conversion (e.g. IN-to-OR expansion), then derive the return type
// and create a flat n-ary RexCall.
final List<RexNode> flatExprs = RexUtil.flatten(exprs, op);
final RelDataType type = rexBuilder.deriveReturnType(op, flatExprs);
return rexBuilder.makeCall(call.getParserPosition(), type, op, flatExprs);
}

private static List<Integer> elseArgs(int count) {
// If list is odd, e.g. [0, 1, 2, 3, 4] we get [1, 3, 4]
// If list is even, e.g. [0, 1, 2, 3, 4, 5] we get [2, 4, 5]
Expand Down
Loading
Loading