diff --git a/core/src/main/java/org/apache/calcite/plan/RelOptUtil.java b/core/src/main/java/org/apache/calcite/plan/RelOptUtil.java index 9b24cf2ad20..b615c31d853 100644 --- a/core/src/main/java/org/apache/calcite/plan/RelOptUtil.java +++ b/core/src/main/java/org/apache/calcite/plan/RelOptUtil.java @@ -70,6 +70,7 @@ import org.apache.calcite.rex.RexExecutorImpl; import org.apache.calcite.rex.RexFieldAccess; import org.apache.calcite.rex.RexInputRef; +import org.apache.calcite.rex.RexLambda; import org.apache.calcite.rex.RexLiteral; import org.apache.calcite.rex.RexLocalRef; import org.apache.calcite.rex.RexNode; @@ -3363,6 +3364,12 @@ private static RexShuttle pushShuttle(final Project project) { @Override public RexNode visitInputRef(RexInputRef ref) { return project.getProjects().get(ref.getIndex()); } + + @Override public RexNode visitLambda(RexLambda lambda) { + // Lambda body references are at a different scope level. + // Do not remap indices inside lambda body against this project. + return lambda; + } }; } @@ -3386,6 +3393,12 @@ private static RexShuttle pushShuttle(final Calc calc) { @Override public RexNode visitInputRef(RexInputRef ref) { return projects.get(ref.getIndex()); } + + @Override public RexNode visitLambda(RexLambda lambda) { + // Lambda body references are at a different scope level. + // Do not remap indices inside lambda body against this calc. + return lambda; + } }; } diff --git a/core/src/main/java/org/apache/calcite/rex/RexBiVisitorImpl.java b/core/src/main/java/org/apache/calcite/rex/RexBiVisitorImpl.java index 5a11bf771e0..0c8f220bff2 100644 --- a/core/src/main/java/org/apache/calcite/rex/RexBiVisitorImpl.java +++ b/core/src/main/java/org/apache/calcite/rex/RexBiVisitorImpl.java @@ -119,8 +119,18 @@ protected RexBiVisitorImpl(boolean deep) { return null; } + /** + * Visits a lambda expression. When {@code deep} is true, recurses into + * the lambda body so that analysis visitors (e.g. InputFinder) can discover + * field references inside the lambda. When {@code deep} is false, returns + * null without recursing — this is the shallow traversal mode used by + * visitors that only need top-level information. + */ @Override public R visitLambda(RexLambda lambda, P arg) { - return null; + if (!deep) { + return null; + } + return lambda.getExpression().accept(this, arg); } @Override public R visitNodeAndFieldIndex(RexNodeAndFieldIndex nodeAndFieldIndex, P arg) { diff --git a/core/src/main/java/org/apache/calcite/rex/RexProgramBuilder.java b/core/src/main/java/org/apache/calcite/rex/RexProgramBuilder.java index 2c8cdfbd9a3..ebeb12aa609 100644 --- a/core/src/main/java/org/apache/calcite/rex/RexProgramBuilder.java +++ b/core/src/main/java/org/apache/calcite/rex/RexProgramBuilder.java @@ -910,8 +910,9 @@ private abstract class RegisterShuttle extends RexShuttle { } @Override public RexNode visitLambda(RexLambda lambda) { - super.visitLambda(lambda); - return registerInternal(lambda); + // Lambda body references are at a different scope level. + // Do not validate or register lambda body indices against this program's input. + return lambda; } } diff --git a/core/src/main/java/org/apache/calcite/rex/RexVisitorImpl.java b/core/src/main/java/org/apache/calcite/rex/RexVisitorImpl.java index 6ebbe92679f..d6d5931d976 100644 --- a/core/src/main/java/org/apache/calcite/rex/RexVisitorImpl.java +++ b/core/src/main/java/org/apache/calcite/rex/RexVisitorImpl.java @@ -118,8 +118,17 @@ protected RexVisitorImpl(boolean deep) { return null; } + /** + * Visits a lambda expression. When {@code deep} is true, recurses into + * the lambda body to analyze its sub-expressions (critical for InputFinder + * to detect field references inside lambda bodies during pushDownJoinConditions). + * When {@code deep} is false, returns null without recursing. + */ @Override public R visitLambda(RexLambda lambda) { - return null; + if (!deep) { + return null; + } + return lambda.getExpression().accept(this); } @Override public R visitLambdaRef(RexLambdaRef lambdaRef) { diff --git a/core/src/main/java/org/apache/calcite/sql/validate/SqlLambdaScope.java b/core/src/main/java/org/apache/calcite/sql/validate/SqlLambdaScope.java index 22003912d45..36e59a1f648 100644 --- a/core/src/main/java/org/apache/calcite/sql/validate/SqlLambdaScope.java +++ b/core/src/main/java/org/apache/calcite/sql/validate/SqlLambdaScope.java @@ -30,8 +30,6 @@ import static com.google.common.base.Preconditions.checkArgument; -import static org.apache.calcite.util.Static.RESOURCE; - /** * Scope for a {@link SqlLambda LAMBDA EXPRESSION}. */ @@ -68,8 +66,7 @@ public boolean isParameter(SqlIdentifier id) { if (found) { return SqlQualified.create(this, 1, null, identifier); } else { - throw validator.newValidationError(identifier, - RESOURCE.paramNotFoundInLambdaExpression(identifier.toString(), lambdaExpr.toString())); + return parent.fullyQualify(identifier); } } diff --git a/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java b/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java index 4622176a073..78d7664aa33 100644 --- a/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java +++ b/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java @@ -5563,11 +5563,16 @@ void setRoot(List inputs) { SqlQualified qualified) { if (nameToNodeMap != null && qualified.prefixLength == 1) { RexNode node = nameToNodeMap.get(qualified.identifier.names.get(0)); - if (node == null) { + if (node != null) { + return Pair.of(node, null); + } + // If the identifier is not found in nameToNodeMap and the current scope + // is a lambda scope, fall through to standard scope resolution to allow + // external references (e.g., t2.v in a JOIN ON lambda expression). + if (!(scope instanceof SqlLambdaScope)) { throw new AssertionError("Unknown identifier '" + qualified.identifier + "' encountered while expanding expression"); } - return Pair.of(node, null); } final SqlNameMatcher nameMatcher = scope.getValidator().getCatalogReader().nameMatcher(); @@ -5586,6 +5591,13 @@ void setRoot(List inputs) { // preserved. final SqlValidatorScope ancestorScope = resolve.scope; boolean isParent = ancestorScope != scope; + // When in a lambda scope, external references to tables that are part + // of the current blackboard's inputs should be resolved locally, not + // as correlation variables. The lambda blackboard inherits inputs from + // its parent blackboard. + if (isParent && scope instanceof SqlLambdaScope && inputs != null) { + isParent = false; + } if ((inputs != null) && !isParent) { final LookupContext rels = new LookupContext(this, inputs, systemFieldList.size()); diff --git a/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java b/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java index 3d7eda0edf5..b70b455d10a 100644 --- a/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java +++ b/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java @@ -8115,7 +8115,10 @@ void testGroupExpressionEquivalenceParams() { /** Test case for * [CALCITE-3679] - * Allow lambda expressions in SQL queries. */ + * Allow lambda expressions in SQL queries. + * [CALCITE-6242] + * Enhance lambda closure parsing. + * */ @Test void testHigherOrderFunction() { final SqlValidatorFixture s = fixture() .withOperatorTable(MockSqlOperatorTable.standard().extend()); @@ -8129,6 +8132,10 @@ void testGroupExpressionEquivalenceParams() { .type("RecordType(INTEGER NOT NULL EXPR$0) NOT NULL"); s.withSql("select HIGHER_ORDER_FUNCTION2(1, () -> 0.1)") .type("RecordType(INTEGER NOT NULL EXPR$0) NOT NULL"); + s.withSql("select HIGHER_ORDER_FUNCTION(1, (x, y) -> x + 1 + ^emp.deptno^) from emp") + .ok(); + s.withSql("select HIGHER_ORDER_FUNCTION(1, (x, y) -> x + 1 + ^deptno^) from emp") + .ok(); // test for type check s.withSql("select HIGHER_ORDER_FUNCTION(1, (x, y) -> ^x + 1^)") @@ -8146,13 +8153,6 @@ void testGroupExpressionEquivalenceParams() { .fails("Cannot apply '(?s).*HIGHER_ORDER_FUNCTION' to arguments of type " + "'HIGHER_ORDER_FUNCTION\\(, ANY>\\)'.*"); - // test for illegal parameters - s.withSql("select HIGHER_ORDER_FUNCTION(1, (x, y) -> x + 1 + ^emp.deptno^) from emp") - .fails("Param 'EMP\\.DEPTNO' not found in lambda expression " - + "'\\(`X`, `Y`\\) -> `X` \\+ 1 \\+ `EMP`\\.`DEPTNO`'"); - s.withSql("select HIGHER_ORDER_FUNCTION(1, (x, y) -> x + 1 + ^deptno^) from emp") - .fails("Param 'DEPTNO' not found in lambda expression " - + "'\\(`X`, `Y`\\) -> `X` \\+ 1 \\+ `DEPTNO`'"); } /** Test case for [CALCITE-7193] diff --git a/core/src/test/resources/sql/lambda.iq b/core/src/test/resources/sql/lambda.iq index 207543ec77c..7d9665cd8d7 100644 --- a/core/src/test/resources/sql/lambda.iq +++ b/core/src/test/resources/sql/lambda.iq @@ -102,3 +102,16 @@ select "EXISTS"(array[array[1, 2], array[3, 4]], x -> x[1] = 1); (1 row) !ok + +# [CALCITE-6242] Enhance lambda closure parsing +select * + from (select array(1, 2, 3) as arr) as t1 inner join + (select 1 as v) as t2 on "EXISTS"(arr, x -> x = t2.v); ++-----------+---+ +| ARR | V | ++-----------+---+ +| [1, 2, 3] | 1 | ++-----------+---+ +(1 row) + +!ok