From b81a5ada29cf7e9000462649993d011241155ce1 Mon Sep 17 00:00:00 2001 From: Terran Date: Fri, 8 May 2026 16:31:32 +0800 Subject: [PATCH 1/4] [CALCITE-6242] The "exists" library function throws a "param not found" error when a column is used in lambda evaluation logic --- .../java/org/apache/calcite/plan/RelOptUtil.java | 13 +++++++++++++ .../org/apache/calcite/rex/RexBiVisitorImpl.java | 5 ++++- .../apache/calcite/rex/RexProgramBuilder.java | 5 +++-- .../org/apache/calcite/rex/RexVisitorImpl.java | 5 ++++- .../calcite/sql/validate/SqlLambdaScope.java | 5 +---- .../calcite/sql2rel/SqlToRelConverter.java | 16 ++++++++++++++-- .../apache/calcite/test/SqlValidatorTest.java | 11 ++++------- core/src/test/resources/sql/lambda.iq | 13 +++++++++++++ 8 files changed, 56 insertions(+), 17 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/plan/RelOptUtil.java b/core/src/main/java/org/apache/calcite/plan/RelOptUtil.java index 9b24cf2ad201..b615c31d8533 100644 --- a/core/src/main/java/org/apache/calcite/plan/RelOptUtil.java +++ b/core/src/main/java/org/apache/calcite/plan/RelOptUtil.java @@ -70,6 +70,7 @@ import org.apache.calcite.rex.RexExecutorImpl; import org.apache.calcite.rex.RexFieldAccess; import org.apache.calcite.rex.RexInputRef; +import org.apache.calcite.rex.RexLambda; import org.apache.calcite.rex.RexLiteral; import org.apache.calcite.rex.RexLocalRef; import org.apache.calcite.rex.RexNode; @@ -3363,6 +3364,12 @@ private static RexShuttle pushShuttle(final Project project) { @Override public RexNode visitInputRef(RexInputRef ref) { return project.getProjects().get(ref.getIndex()); } + + @Override public RexNode visitLambda(RexLambda lambda) { + // Lambda body references are at a different scope level. + // Do not remap indices inside lambda body against this project. + return lambda; + } }; } @@ -3386,6 +3393,12 @@ private static RexShuttle pushShuttle(final Calc calc) { @Override public RexNode visitInputRef(RexInputRef ref) { return projects.get(ref.getIndex()); } + + @Override public RexNode visitLambda(RexLambda lambda) { + // Lambda body references are at a different scope level. + // Do not remap indices inside lambda body against this calc. + return lambda; + } }; } diff --git a/core/src/main/java/org/apache/calcite/rex/RexBiVisitorImpl.java b/core/src/main/java/org/apache/calcite/rex/RexBiVisitorImpl.java index 5a11bf771e0e..3477d045769c 100644 --- a/core/src/main/java/org/apache/calcite/rex/RexBiVisitorImpl.java +++ b/core/src/main/java/org/apache/calcite/rex/RexBiVisitorImpl.java @@ -120,7 +120,10 @@ protected RexBiVisitorImpl(boolean deep) { } @Override public R visitLambda(RexLambda lambda, P arg) { - return null; + if (!deep) { + return null; + } + return lambda.getExpression().accept(this, arg); } @Override public R visitNodeAndFieldIndex(RexNodeAndFieldIndex nodeAndFieldIndex, P arg) { diff --git a/core/src/main/java/org/apache/calcite/rex/RexProgramBuilder.java b/core/src/main/java/org/apache/calcite/rex/RexProgramBuilder.java index 2c8cdfbd9a39..ebeb12aa609e 100644 --- a/core/src/main/java/org/apache/calcite/rex/RexProgramBuilder.java +++ b/core/src/main/java/org/apache/calcite/rex/RexProgramBuilder.java @@ -910,8 +910,9 @@ private abstract class RegisterShuttle extends RexShuttle { } @Override public RexNode visitLambda(RexLambda lambda) { - super.visitLambda(lambda); - return registerInternal(lambda); + // Lambda body references are at a different scope level. + // Do not validate or register lambda body indices against this program's input. + return lambda; } } diff --git a/core/src/main/java/org/apache/calcite/rex/RexVisitorImpl.java b/core/src/main/java/org/apache/calcite/rex/RexVisitorImpl.java index 6ebbe92679f6..e1cff744a03c 100644 --- a/core/src/main/java/org/apache/calcite/rex/RexVisitorImpl.java +++ b/core/src/main/java/org/apache/calcite/rex/RexVisitorImpl.java @@ -119,7 +119,10 @@ protected RexVisitorImpl(boolean deep) { } @Override public R visitLambda(RexLambda lambda) { - return null; + if (!deep) { + return null; + } + return lambda.getExpression().accept(this); } @Override public R visitLambdaRef(RexLambdaRef lambdaRef) { diff --git a/core/src/main/java/org/apache/calcite/sql/validate/SqlLambdaScope.java b/core/src/main/java/org/apache/calcite/sql/validate/SqlLambdaScope.java index 22003912d457..36e59a1f648a 100644 --- a/core/src/main/java/org/apache/calcite/sql/validate/SqlLambdaScope.java +++ b/core/src/main/java/org/apache/calcite/sql/validate/SqlLambdaScope.java @@ -30,8 +30,6 @@ import static com.google.common.base.Preconditions.checkArgument; -import static org.apache.calcite.util.Static.RESOURCE; - /** * Scope for a {@link SqlLambda LAMBDA EXPRESSION}. */ @@ -68,8 +66,7 @@ public boolean isParameter(SqlIdentifier id) { if (found) { return SqlQualified.create(this, 1, null, identifier); } else { - throw validator.newValidationError(identifier, - RESOURCE.paramNotFoundInLambdaExpression(identifier.toString(), lambdaExpr.toString())); + return parent.fullyQualify(identifier); } } diff --git a/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java b/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java index 4622176a0733..78d7664aa335 100644 --- a/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java +++ b/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java @@ -5563,11 +5563,16 @@ void setRoot(List inputs) { SqlQualified qualified) { if (nameToNodeMap != null && qualified.prefixLength == 1) { RexNode node = nameToNodeMap.get(qualified.identifier.names.get(0)); - if (node == null) { + if (node != null) { + return Pair.of(node, null); + } + // If the identifier is not found in nameToNodeMap and the current scope + // is a lambda scope, fall through to standard scope resolution to allow + // external references (e.g., t2.v in a JOIN ON lambda expression). + if (!(scope instanceof SqlLambdaScope)) { throw new AssertionError("Unknown identifier '" + qualified.identifier + "' encountered while expanding expression"); } - return Pair.of(node, null); } final SqlNameMatcher nameMatcher = scope.getValidator().getCatalogReader().nameMatcher(); @@ -5586,6 +5591,13 @@ void setRoot(List inputs) { // preserved. final SqlValidatorScope ancestorScope = resolve.scope; boolean isParent = ancestorScope != scope; + // When in a lambda scope, external references to tables that are part + // of the current blackboard's inputs should be resolved locally, not + // as correlation variables. The lambda blackboard inherits inputs from + // its parent blackboard. + if (isParent && scope instanceof SqlLambdaScope && inputs != null) { + isParent = false; + } if ((inputs != null) && !isParent) { final LookupContext rels = new LookupContext(this, inputs, systemFieldList.size()); diff --git a/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java b/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java index 3d7eda0edf55..6f7b7e1da58f 100644 --- a/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java +++ b/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java @@ -8129,6 +8129,10 @@ void testGroupExpressionEquivalenceParams() { .type("RecordType(INTEGER NOT NULL EXPR$0) NOT NULL"); s.withSql("select HIGHER_ORDER_FUNCTION2(1, () -> 0.1)") .type("RecordType(INTEGER NOT NULL EXPR$0) NOT NULL"); + s.withSql("select HIGHER_ORDER_FUNCTION(1, (x, y) -> x + 1 + ^emp.deptno^) from emp") + .ok(); + s.withSql("select HIGHER_ORDER_FUNCTION(1, (x, y) -> x + 1 + ^deptno^) from emp") + .ok(); // test for type check s.withSql("select HIGHER_ORDER_FUNCTION(1, (x, y) -> ^x + 1^)") @@ -8146,13 +8150,6 @@ void testGroupExpressionEquivalenceParams() { .fails("Cannot apply '(?s).*HIGHER_ORDER_FUNCTION' to arguments of type " + "'HIGHER_ORDER_FUNCTION\\(, ANY>\\)'.*"); - // test for illegal parameters - s.withSql("select HIGHER_ORDER_FUNCTION(1, (x, y) -> x + 1 + ^emp.deptno^) from emp") - .fails("Param 'EMP\\.DEPTNO' not found in lambda expression " - + "'\\(`X`, `Y`\\) -> `X` \\+ 1 \\+ `EMP`\\.`DEPTNO`'"); - s.withSql("select HIGHER_ORDER_FUNCTION(1, (x, y) -> x + 1 + ^deptno^) from emp") - .fails("Param 'DEPTNO' not found in lambda expression " - + "'\\(`X`, `Y`\\) -> `X` \\+ 1 \\+ `DEPTNO`'"); } /** Test case for [CALCITE-7193] diff --git a/core/src/test/resources/sql/lambda.iq b/core/src/test/resources/sql/lambda.iq index 207543ec77cb..79493fe19018 100644 --- a/core/src/test/resources/sql/lambda.iq +++ b/core/src/test/resources/sql/lambda.iq @@ -102,3 +102,16 @@ select "EXISTS"(array[array[1, 2], array[3, 4]], x -> x[1] = 1); (1 row) !ok + + +select * + from (select array(1, 2, 3) as arr) as t1 inner join + (select 1 as v) as t2 on "EXISTS"(arr, x -> x = t2.v); ++-----------+---+ +| ARR | V | ++-----------+---+ +| [1, 2, 3] | 1 | ++-----------+---+ +(1 row) + +!ok From 265a32d0fd40f2c3d9da5aea44830877aa5aef57 Mon Sep 17 00:00:00 2001 From: Terran Date: Fri, 8 May 2026 17:02:03 +0800 Subject: [PATCH 2/4] [CALCITE-6242] Remove blank lines --- core/src/test/resources/sql/lambda.iq | 1 - 1 file changed, 1 deletion(-) diff --git a/core/src/test/resources/sql/lambda.iq b/core/src/test/resources/sql/lambda.iq index 79493fe19018..92c56a94f63e 100644 --- a/core/src/test/resources/sql/lambda.iq +++ b/core/src/test/resources/sql/lambda.iq @@ -103,7 +103,6 @@ select "EXISTS"(array[array[1, 2], array[3, 4]], x -> x[1] = 1); !ok - select * from (select array(1, 2, 3) as arr) as t1 inner join (select 1 as v) as t2 on "EXISTS"(arr, x -> x = t2.v); From ca022654d70144a3856e7ee4384e8a0523188b6b Mon Sep 17 00:00:00 2001 From: Terran Date: Sat, 9 May 2026 11:01:52 +0800 Subject: [PATCH 3/4] [CALCITE-6242] Add comments --- .../main/java/org/apache/calcite/rex/RexBiVisitorImpl.java | 7 +++++++ .../main/java/org/apache/calcite/rex/RexVisitorImpl.java | 6 ++++++ .../java/org/apache/calcite/test/SqlValidatorTest.java | 6 +++++- core/src/test/resources/sql/lambda.iq | 2 ++ 4 files changed, 20 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/org/apache/calcite/rex/RexBiVisitorImpl.java b/core/src/main/java/org/apache/calcite/rex/RexBiVisitorImpl.java index 3477d045769c..0c8f220bff29 100644 --- a/core/src/main/java/org/apache/calcite/rex/RexBiVisitorImpl.java +++ b/core/src/main/java/org/apache/calcite/rex/RexBiVisitorImpl.java @@ -119,6 +119,13 @@ protected RexBiVisitorImpl(boolean deep) { return null; } + /** + * Visits a lambda expression. When {@code deep} is true, recurses into + * the lambda body so that analysis visitors (e.g. InputFinder) can discover + * field references inside the lambda. When {@code deep} is false, returns + * null without recursing — this is the shallow traversal mode used by + * visitors that only need top-level information. + */ @Override public R visitLambda(RexLambda lambda, P arg) { if (!deep) { return null; diff --git a/core/src/main/java/org/apache/calcite/rex/RexVisitorImpl.java b/core/src/main/java/org/apache/calcite/rex/RexVisitorImpl.java index e1cff744a03c..d6d5931d9768 100644 --- a/core/src/main/java/org/apache/calcite/rex/RexVisitorImpl.java +++ b/core/src/main/java/org/apache/calcite/rex/RexVisitorImpl.java @@ -118,6 +118,12 @@ protected RexVisitorImpl(boolean deep) { return null; } + /** + * Visits a lambda expression. When {@code deep} is true, recurses into + * the lambda body to analyze its sub-expressions (critical for InputFinder + * to detect field references inside lambda bodies during pushDownJoinConditions). + * When {@code deep} is false, returns null without recursing. + */ @Override public R visitLambda(RexLambda lambda) { if (!deep) { return null; diff --git a/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java b/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java index 6f7b7e1da58f..74621ecf6c38 100644 --- a/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java +++ b/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java @@ -8115,7 +8115,11 @@ void testGroupExpressionEquivalenceParams() { /** Test case for * [CALCITE-3679] - * Allow lambda expressions in SQL queries. */ + * Allow lambda expressions in SQL queries. + * [CALCITE-6242] + * The "exists" library function throws a "param not found" error when a + * column is used in lambda evaluation logic. + * */ @Test void testHigherOrderFunction() { final SqlValidatorFixture s = fixture() .withOperatorTable(MockSqlOperatorTable.standard().extend()); diff --git a/core/src/test/resources/sql/lambda.iq b/core/src/test/resources/sql/lambda.iq index 92c56a94f63e..32339638f607 100644 --- a/core/src/test/resources/sql/lambda.iq +++ b/core/src/test/resources/sql/lambda.iq @@ -103,6 +103,8 @@ select "EXISTS"(array[array[1, 2], array[3, 4]], x -> x[1] = 1); !ok +# [CALCITE-6242] The "exists" library function throws a "param not found" +# error when a column is used in lambda evaluation logic select * from (select array(1, 2, 3) as arr) as t1 inner join (select 1 as v) as t2 on "EXISTS"(arr, x -> x = t2.v); From a705f79f9789741b80070ef1c208870debe5edcf Mon Sep 17 00:00:00 2001 From: Terran Date: Sat, 9 May 2026 19:19:50 +0800 Subject: [PATCH 4/4] [CALCITE-6242] Modify comments --- .../test/java/org/apache/calcite/test/SqlValidatorTest.java | 3 +-- core/src/test/resources/sql/lambda.iq | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java b/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java index 74621ecf6c38..b70b455d10ac 100644 --- a/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java +++ b/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java @@ -8117,8 +8117,7 @@ void testGroupExpressionEquivalenceParams() { * [CALCITE-3679] * Allow lambda expressions in SQL queries. * [CALCITE-6242] - * The "exists" library function throws a "param not found" error when a - * column is used in lambda evaluation logic. + * Enhance lambda closure parsing. * */ @Test void testHigherOrderFunction() { final SqlValidatorFixture s = fixture() diff --git a/core/src/test/resources/sql/lambda.iq b/core/src/test/resources/sql/lambda.iq index 32339638f607..7d9665cd8d73 100644 --- a/core/src/test/resources/sql/lambda.iq +++ b/core/src/test/resources/sql/lambda.iq @@ -103,8 +103,7 @@ select "EXISTS"(array[array[1, 2], array[3, 4]], x -> x[1] = 1); !ok -# [CALCITE-6242] The "exists" library function throws a "param not found" -# error when a column is used in lambda evaluation logic +# [CALCITE-6242] Enhance lambda closure parsing select * from (select array(1, 2, 3) as arr) as t1 inner join (select 1 as v) as t2 on "EXISTS"(arr, x -> x = t2.v);