[CALCITE-6242] Enhance lambda closure parsing

core/src/main/java/org/apache/calcite/plan/RelOptUtil.java

@@ -70,6 +70,7 @@
 import org.apache.calcite.rex.RexExecutorImpl;
 import org.apache.calcite.rex.RexFieldAccess;
 import org.apache.calcite.rex.RexInputRef;
+import org.apache.calcite.rex.RexLambda;
 import org.apache.calcite.rex.RexLiteral;
 import org.apache.calcite.rex.RexLocalRef;
 import org.apache.calcite.rex.RexNode;
@@ -3363,6 +3364,12 @@ private static RexShuttle pushShuttle(final Project project) {
       @Override public RexNode visitInputRef(RexInputRef ref) {
         return project.getProjects().get(ref.getIndex());
       }
+
+      @Override public RexNode visitLambda(RexLambda lambda) {
+        // Lambda body references are at a different scope level.
+        // Do not remap indices inside lambda body against this project.
+        return lambda;
+      }
     };
   }
 
@@ -3386,6 +3393,12 @@ private static RexShuttle pushShuttle(final Calc calc) {
       @Override public RexNode visitInputRef(RexInputRef ref) {
         return projects.get(ref.getIndex());
       }
+
+      @Override public RexNode visitLambda(RexLambda lambda) {
+        // Lambda body references are at a different scope level.
+        // Do not remap indices inside lambda body against this calc.
+        return lambda;
+      }
     };
   }
 

core/src/main/java/org/apache/calcite/rex/RexBiVisitorImpl.java

@@ -119,8 +119,18 @@ protected RexBiVisitorImpl(boolean deep) {
     return null;
   }
 
+  /**
+   * Visits a lambda expression. When {@code deep} is true, recurses into
+   * the lambda body so that analysis visitors (e.g. InputFinder) can discover
+   * field references inside the lambda. When {@code deep} is false, returns
+   * null without recursing — this is the shallow traversal mode used by
+   * visitors that only need top-level information.
+   */
   @Override public R visitLambda(RexLambda lambda, P arg) {
-    return null;
+    if (!deep) {
+      return null;
+    }
+    return lambda.getExpression().accept(this, arg);
   }
 
   @Override public R visitNodeAndFieldIndex(RexNodeAndFieldIndex nodeAndFieldIndex, P arg) {

core/src/main/java/org/apache/calcite/rex/RexProgramBuilder.java

@@ -910,8 +910,9 @@ private abstract class RegisterShuttle extends RexShuttle {
     }
 
     @Override public RexNode visitLambda(RexLambda lambda) {
-      super.visitLambda(lambda);
-      return registerInternal(lambda);
+      // Lambda body references are at a different scope level.
+      // Do not validate or register lambda body indices against this program's input.
+      return lambda;
     }
   }
 

core/src/main/java/org/apache/calcite/rex/RexVisitorImpl.java

@@ -118,8 +118,17 @@ protected RexVisitorImpl(boolean deep) {
     return null;
   }
 
+  /**
+   * Visits a lambda expression. When {@code deep} is true, recurses into
+   * the lambda body to analyze its sub-expressions (critical for InputFinder
+   * to detect field references inside lambda bodies during pushDownJoinConditions).
+   * When {@code deep} is false, returns null without recursing.
+   */
   @Override public R visitLambda(RexLambda lambda) {
-    return null;
+    if (!deep) {
+      return null;
+    }
+    return lambda.getExpression().accept(this);
   }
 
   @Override public R visitLambdaRef(RexLambdaRef lambdaRef) {

core/src/main/java/org/apache/calcite/sql/validate/SqlLambdaScope.java

@@ -30,8 +30,6 @@
 
 import static com.google.common.base.Preconditions.checkArgument;
 
-import static org.apache.calcite.util.Static.RESOURCE;
-
 /**
  * Scope for a {@link SqlLambda LAMBDA EXPRESSION}.
  */
@@ -68,8 +66,7 @@ public boolean isParameter(SqlIdentifier id) {
     if (found) {
       return SqlQualified.create(this, 1, null, identifier);
     } else {
-      throw validator.newValidationError(identifier,
-          RESOURCE.paramNotFoundInLambdaExpression(identifier.toString(), lambdaExpr.toString()));
+      return parent.fullyQualify(identifier);
     }
   }
 

core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java

@@ -5563,11 +5563,16 @@ void setRoot(List<RelNode> inputs) {
         SqlQualified qualified) {
       if (nameToNodeMap != null && qualified.prefixLength == 1) {
         RexNode node = nameToNodeMap.get(qualified.identifier.names.get(0));
-        if (node == null) {
+        if (node != null) {
+          return Pair.of(node, null);
+        }
+        // If the identifier is not found in nameToNodeMap and the current scope
+        // is a lambda scope, fall through to standard scope resolution to allow
+        // external references (e.g., t2.v in a JOIN ON lambda expression).
+        if (!(scope instanceof SqlLambdaScope)) {
           throw new AssertionError("Unknown identifier '" + qualified.identifier
               + "' encountered while expanding expression");
         }
-        return Pair.of(node, null);
       }
       final SqlNameMatcher nameMatcher =
           scope.getValidator().getCatalogReader().nameMatcher();
@@ -5586,6 +5591,13 @@ void setRoot(List<RelNode> inputs) {
       // preserved.
       final SqlValidatorScope ancestorScope = resolve.scope;
       boolean isParent = ancestorScope != scope;
+      // When in a lambda scope, external references to tables that are part
+      // of the current blackboard's inputs should be resolved locally, not
+      // as correlation variables. The lambda blackboard inherits inputs from
+      // its parent blackboard.
+      if (isParent && scope instanceof SqlLambdaScope && inputs != null) {
+        isParent = false;
+      }
       if ((inputs != null) && !isParent) {
         final LookupContext rels =
             new LookupContext(this, inputs, systemFieldList.size());

core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java

@@ -8115,7 +8115,10 @@ void testGroupExpressionEquivalenceParams() {
 
   /** Test case for
    * <a href="https://issues.apache.org/jira/browse/CALCITE-3679">[CALCITE-3679]
-   * Allow lambda expressions in SQL queries</a>. */
+   * Allow lambda expressions in SQL queries</a>.
+   * <a href="https://issues.apache.org/jira/browse/CALCITE-6242">[CALCITE-6242]
+   * Enhance lambda closure parsing</a>.
+   * */
   @Test void testHigherOrderFunction() {
     final SqlValidatorFixture s = fixture()
         .withOperatorTable(MockSqlOperatorTable.standard().extend());
@@ -8129,6 +8132,10 @@ void testGroupExpressionEquivalenceParams() {
         .type("RecordType(INTEGER NOT NULL EXPR$0) NOT NULL");
     s.withSql("select HIGHER_ORDER_FUNCTION2(1, () -> 0.1)")
         .type("RecordType(INTEGER NOT NULL EXPR$0) NOT NULL");
+    s.withSql("select HIGHER_ORDER_FUNCTION(1, (x, y) -> x + 1 + ^emp.deptno^) from emp")
+        .ok();
+    s.withSql("select HIGHER_ORDER_FUNCTION(1, (x, y) -> x + 1 + ^deptno^) from emp")
+        .ok();
 
     // test for type check
     s.withSql("select HIGHER_ORDER_FUNCTION(1, (x, y) -> ^x + 1^)")
@@ -8146,13 +8153,6 @@ void testGroupExpressionEquivalenceParams() {
         .fails("Cannot apply '(?s).*HIGHER_ORDER_FUNCTION' to arguments of type "
             + "'HIGHER_ORDER_FUNCTION\\(<INTEGER>, <FUNCTION\\(ANY, ANY, ANY\\) -> ANY>\\)'.*");
 
-    // test for illegal parameters
-    s.withSql("select HIGHER_ORDER_FUNCTION(1, (x, y) -> x + 1 + ^emp.deptno^) from emp")
-        .fails("Param 'EMP\\.DEPTNO' not found in lambda expression "
-            + "'\\(`X`, `Y`\\) -> `X` \\+ 1 \\+ `EMP`\\.`DEPTNO`'");
-    s.withSql("select HIGHER_ORDER_FUNCTION(1, (x, y) -> x + 1 + ^deptno^) from emp")
-        .fails("Param 'DEPTNO' not found in lambda expression "
-            + "'\\(`X`, `Y`\\) -> `X` \\+ 1 \\+ `DEPTNO`'");
   }
 
   /** Test case for <a href="https://issues.apache.org/jira/browse/CALCITE-7193">[CALCITE-7193]

core/src/test/resources/sql/lambda.iq

@@ -102,3 +102,16 @@ select "EXISTS"(array[array[1, 2], array[3, 4]], x -> x[1] = 1);
 (1 row)
 
 !ok
+
+# [CALCITE-6242] Enhance lambda closure parsing
+select *
+ from (select array(1, 2, 3) as arr) as t1 inner join
+     (select 1 as v) as t2 on "EXISTS"(arr, x -> x = t2.v);
++-----------+---+
+| ARR       | V |
++-----------+---+
+| [1, 2, 3] | 1 |
++-----------+---+
+(1 row)
+
+!ok