TEDEFO-4919 Add for-return with sequence body and return distinct

src/main/java/eu/europa/ted/efx/interfaces/ScriptGenerator.java

@@ -224,9 +224,72 @@ public BooleanExpression composeAnySatisfies(
   public <T extends TypedExpression> T composeConditionalExpression(BooleanExpression condition,
       T whenTrue, T whenFalse, Class<T> type);
 
+  /**
+   * Composes a "for ... return" expression that maps each iteration to a scalar value,
+   * collecting the results into a sequence (a map operation).
+   *
+   * EFX:   for text:$x in ['a', 'b', 'c'] return concat($x, '!')
+   * XPath: for $x in ('a','b','c') return concat($x, '!')
+   *
+   * Java:
+   *   List<String> result = new ArrayList<>();
+   *   for (String x : List.of("a", "b", "c")) {
+   *       result.add(x + "!");
+   *   }
+   *   // result = ["a!", "b!", "c!"]
+   *
+   * JavaScript:
+   *   const result = ["a", "b", "c"].map(x => x + "!");
+   *
+   * Python:
+   *   result = [x + "!" for x in ["a", "b", "c"]]
+   *
+   * @param iterators the iterator list (one or more typed iterators)
+   * @param expression the scalar expression evaluated per iteration
+   * @param targetListType the class of the resulting sequence expression
+   * @return the target-language script for the for-return expression
+   */
   public <T extends SequenceExpression> T composeForExpression(
       IteratorListExpression iterators, ScalarExpression expression, Class<T> targetListType);
 
+  /**
+   * Composes a "for ... return" expression where the return body is a sequence,
+   * concatenating all results into a single flat sequence (a flatMap operation).
+   *
+   * Unlike {@link #composeForExpression} where the body is a scalar (map), here the
+   * body produces a sequence per iteration. The results are concatenated without
+   * removing duplicates.
+   *
+   * EFX:   for number:$x in [1, 2, 3] return [$x, $x * 10]
+   * XPath: for $x in (1, 2, 3) return ($x, $x * 10)
+   *        (XPath flattens sequences automatically)
+   *
+   * Java:
+   *   List<Integer> result = new ArrayList<>();
+   *   for (int x : List.of(1, 2, 3)) {
+   *       result.addAll(List.of(x, x * 10));
+   *   }
+   *   // result = [1, 10, 2, 20, 3, 30]
+   *
+   * JavaScript:
+   *   const result = [1, 2, 3].flatMap(x => [x, x * 10]);
+   *
+   * Python:
+   *   result = [item for x in [1, 2, 3] for item in [x, x * 10]]
+   *
+   * The key difference from composeForExpression is add() vs addAll():
+   * composeForExpression (scalar body) adds one element per iteration,
+   * composeForExpression (sequence body) adds all elements of a
+   * sub-sequence per iteration.
+   *
+   * @param iterators the iterator list (one or more typed iterators)
+   * @param sequenceExpression the sequence expression evaluated per iteration
+   * @param targetListType the class of the resulting sequence expression
+   * @return the target-language script for the for expression
+   */
+  public <T extends SequenceExpression> T composeForExpression(
+      IteratorListExpression iterators, SequenceExpression sequenceExpression, Class<T> targetListType);
+
   public IteratorExpression composeIteratorExpression(Expression variableDeclarationExpression, SequenceExpression sourceList);
 
   public IteratorListExpression composeIteratorList(List<IteratorExpression> iterators);

src/main/java/eu/europa/ted/efx/sdk2/EfxExpressionTranslatorV2.java

@@ -1032,7 +1032,7 @@ public void enterStringSequenceFromIteration(StringSequenceFromIterationContext
 
   @Override
   public void exitStringSequenceFromIteration(StringSequenceFromIterationContext ctx) {
-    this.exitIterationExpression(StringExpression.class, StringSequenceExpression.class);
+    this.exitIterationExpression(StringExpression.class, StringSequenceExpression.class, ctx.Distinct() != null);
     this.stack.popStackFrame(); // Iteration variables are local to the iteration
   }
 
@@ -1043,7 +1043,7 @@ public void enterNumericSequenceFromIteration(NumericSequenceFromIterationContex
 
   @Override
   public void exitNumericSequenceFromIteration(NumericSequenceFromIterationContext ctx) {
-    this.exitIterationExpression(NumericExpression.class, NumericSequenceExpression.class);
+    this.exitIterationExpression(NumericExpression.class, NumericSequenceExpression.class, ctx.Distinct() != null);
     this.stack.popStackFrame(); // Iteration variables are local to the iteration
   }
 
@@ -1054,7 +1054,7 @@ public void enterBooleanSequenceFromIteration(BooleanSequenceFromIterationContex
 
   @Override
   public void exitBooleanSequenceFromIteration(BooleanSequenceFromIterationContext ctx) {
-    this.exitIterationExpression(BooleanExpression.class, BooleanSequenceExpression.class);
+    this.exitIterationExpression(BooleanExpression.class, BooleanSequenceExpression.class, ctx.Distinct() != null);
     this.stack.popStackFrame(); // Iteration variables are local to the iteration
   }
 
@@ -1065,7 +1065,7 @@ public void enterDateSequenceFromIteration(DateSequenceFromIterationContext ctx)
 
   @Override
   public void exitDateSequenceFromIteration(DateSequenceFromIterationContext ctx) {
-    this.exitIterationExpression(DateExpression.class, DateSequenceExpression.class);
+    this.exitIterationExpression(DateExpression.class, DateSequenceExpression.class, ctx.Distinct() != null);
     this.stack.popStackFrame(); // Iteration variables are local to the iteration
   }
 
@@ -1076,7 +1076,7 @@ public void enterTimeSequenceFromIteration(TimeSequenceFromIterationContext ctx)
 
   @Override
   public void exitTimeSequenceFromIteration(TimeSequenceFromIterationContext ctx) {
-    this.exitIterationExpression(TimeExpression.class, TimeSequenceExpression.class);
+    this.exitIterationExpression(TimeExpression.class, TimeSequenceExpression.class, ctx.Distinct() != null);
     this.stack.popStackFrame(); // Iteration variables are local to the iteration
   }
 
@@ -1087,10 +1087,78 @@ public void enterDurationSequenceFromIteration(DurationSequenceFromIterationCont
 
   @Override
   public void exitDurationSequenceFromIteration(DurationSequenceFromIterationContext ctx) {
-    this.exitIterationExpression(DurationExpression.class, DurationSequenceExpression.class);
+    this.exitIterationExpression(DurationExpression.class, DurationSequenceExpression.class, ctx.Distinct() != null);
     this.stack.popStackFrame(); // Iteration variables are local to the iteration
   }
 
+  // for ... return <sequence> (concatenated iterations, i.e. flatMap)
+
+  @Override
+  public void enterStringSequenceFromConcatenatedIterations(StringSequenceFromConcatenatedIterationsContext ctx) {
+    this.stack.pushStackFrame();
+  }
+
+  @Override
+  public void exitStringSequenceFromConcatenatedIterations(StringSequenceFromConcatenatedIterationsContext ctx) {
+    this.exitConcatenatedIterationExpression(StringSequenceExpression.class, ctx.Distinct() != null);
+    this.stack.popStackFrame();
+  }
+
+  @Override
+  public void enterBooleanSequenceFromConcatenatedIterations(BooleanSequenceFromConcatenatedIterationsContext ctx) {
+    this.stack.pushStackFrame();
+  }
+
+  @Override
+  public void exitBooleanSequenceFromConcatenatedIterations(BooleanSequenceFromConcatenatedIterationsContext ctx) {
+    this.exitConcatenatedIterationExpression(BooleanSequenceExpression.class, ctx.Distinct() != null);
+    this.stack.popStackFrame();
+  }
+
+  @Override
+  public void enterNumericSequenceFromConcatenatedIterations(NumericSequenceFromConcatenatedIterationsContext ctx) {
+    this.stack.pushStackFrame();
+  }
+
+  @Override
+  public void exitNumericSequenceFromConcatenatedIterations(NumericSequenceFromConcatenatedIterationsContext ctx) {
+    this.exitConcatenatedIterationExpression(NumericSequenceExpression.class, ctx.Distinct() != null);
+    this.stack.popStackFrame();
+  }
+
+  @Override
+  public void enterDateSequenceFromConcatenatedIterations(DateSequenceFromConcatenatedIterationsContext ctx) {
+    this.stack.pushStackFrame();
+  }
+
+  @Override
+  public void exitDateSequenceFromConcatenatedIterations(DateSequenceFromConcatenatedIterationsContext ctx) {
+    this.exitConcatenatedIterationExpression(DateSequenceExpression.class, ctx.Distinct() != null);
+    this.stack.popStackFrame();
+  }
+
+  @Override
+  public void enterTimeSequenceFromConcatenatedIterations(TimeSequenceFromConcatenatedIterationsContext ctx) {
+    this.stack.pushStackFrame();
+  }
+
+  @Override
+  public void exitTimeSequenceFromConcatenatedIterations(TimeSequenceFromConcatenatedIterationsContext ctx) {
+    this.exitConcatenatedIterationExpression(TimeSequenceExpression.class, ctx.Distinct() != null);
+    this.stack.popStackFrame();
+  }
+
+  @Override
+  public void enterDurationSequenceFromConcatenatedIterations(DurationSequenceFromConcatenatedIterationsContext ctx) {
+    this.stack.pushStackFrame();
+  }
+
+  @Override
+  public void exitDurationSequenceFromConcatenatedIterations(DurationSequenceFromConcatenatedIterationsContext ctx) {
+    this.exitConcatenatedIterationExpression(DurationSequenceExpression.class, ctx.Distinct() != null);
+    this.stack.popStackFrame();
+  }
+
   public <T1 extends ScalarExpression, T2 extends SequenceExpression> void exitIteratorExpression(String variableName,
       Class<T1> variableType, Class<T2> listType) {
     Expression declarationExpression = this.script.composeVariableDeclaration(variableName, variableType);
@@ -1102,13 +1170,26 @@ public <T1 extends ScalarExpression, T2 extends SequenceExpression> void exitIte
     this.stack.push(this.script.composeIteratorExpression(variable.declarationExpression, initialisationExpression));
   }
 
-  public <T extends ScalarExpression> void exitIterationExpression(
-      Class<T> expressionType,
-      Class<? extends SequenceExpression> targetListType) {
+  @SuppressWarnings("unchecked")
+  public <T extends ScalarExpression, L extends SequenceExpression> void exitIterationExpression(
+      Class<T> expressionType, Class<L> targetListType, boolean distinct) {
     T expression = this.stack.pop(expressionType);
     IteratorListExpression iterators = this.stack.pop(IteratorListExpression.class);
-    this.stack
-        .push(this.script.composeForExpression(iterators, expression, targetListType));
+    L result = this.script.composeForExpression(iterators, expression, targetListType);
+    if (distinct) {
+      result = this.script.composeDistinctValuesFunction(result, targetListType);
+    }
+    this.stack.push(result);
+  }
+
+  public <T extends SequenceExpression> void exitConcatenatedIterationExpression(Class<T> sequenceType, boolean distinct) {
+    T sequenceExpression = this.stack.pop(sequenceType);
+    IteratorListExpression iterators = this.stack.pop(IteratorListExpression.class);
+    T result = this.script.composeForExpression(iterators, sequenceExpression, sequenceType);
+    if (distinct) {
+      result = this.script.composeDistinctValuesFunction(result, sequenceType);
+    }
+    this.stack.push(result);
   }
 
   // #endregion Iterators -----------------------------------------------------
@@ -2996,6 +3077,12 @@ public void exitScalarFromFieldReference(ScalarFromFieldReferenceContext ctx) {
             this.rewriter.insertBefore(ctx.getStart(), "(" + fieldType + "*)");
             return;
           }
+          // If inside a for-return body, auto-insert sequence cast so the re-parsed
+          // expression matches the ConcatenatedIterations rule (flatMap semantics).
+          if (hasParentContextOfType(ctx, LateBoundSequenceFromIterationContext.class)) {
+            this.rewriter.insertBefore(ctx.getStart(), "(" + fieldType + "*)");
+            return;
+          }
           throw TypeMismatchException.fieldMayRepeat(fieldId, this.efxContext.symbol());
         }
       }
@@ -3502,6 +3589,78 @@ public void exitLateBoundSequenceFromIteration(LateBoundSequenceFromIterationCon
       this.stack.popStackFrame();
     }
 
+    // for ... return <sequence> (concatenated iterations)
+
+    @Override
+    public void enterStringSequenceFromConcatenatedIterations(StringSequenceFromConcatenatedIterationsContext ctx) {
+      this.stack.pushStackFrame();
+    }
+
+    @Override
+    public void exitStringSequenceFromConcatenatedIterations(StringSequenceFromConcatenatedIterationsContext ctx) {
+      this.stack.popStackFrame();
+    }
+
+    @Override
+    public void enterNumericSequenceFromConcatenatedIterations(NumericSequenceFromConcatenatedIterationsContext ctx) {
+      this.stack.pushStackFrame();
+    }
+
+    @Override
+    public void exitNumericSequenceFromConcatenatedIterations(NumericSequenceFromConcatenatedIterationsContext ctx) {
+      this.stack.popStackFrame();
+    }
+
+    @Override
+    public void enterBooleanSequenceFromConcatenatedIterations(BooleanSequenceFromConcatenatedIterationsContext ctx) {
+      this.stack.pushStackFrame();
+    }
+
+    @Override
+    public void exitBooleanSequenceFromConcatenatedIterations(BooleanSequenceFromConcatenatedIterationsContext ctx) {
+      this.stack.popStackFrame();
+    }
+
+    @Override
+    public void enterDateSequenceFromConcatenatedIterations(DateSequenceFromConcatenatedIterationsContext ctx) {
+      this.stack.pushStackFrame();
+    }
+
+    @Override
+    public void exitDateSequenceFromConcatenatedIterations(DateSequenceFromConcatenatedIterationsContext ctx) {
+      this.stack.popStackFrame();
+    }
+
+    @Override
+    public void enterTimeSequenceFromConcatenatedIterations(TimeSequenceFromConcatenatedIterationsContext ctx) {
+      this.stack.pushStackFrame();
+    }
+
+    @Override
+    public void exitTimeSequenceFromConcatenatedIterations(TimeSequenceFromConcatenatedIterationsContext ctx) {
+      this.stack.popStackFrame();
+    }
+
+    @Override
+    public void enterDurationSequenceFromConcatenatedIterations(DurationSequenceFromConcatenatedIterationsContext ctx) {
+      this.stack.pushStackFrame();
+    }
+
+    @Override
+    public void exitDurationSequenceFromConcatenatedIterations(DurationSequenceFromConcatenatedIterationsContext ctx) {
+      this.stack.popStackFrame();
+    }
+
+    @Override
+    public void enterLateBoundSequenceFromConcatenatedIterations(LateBoundSequenceFromConcatenatedIterationsContext ctx) {
+      this.stack.pushStackFrame();
+    }
+
+    @Override
+    public void exitLateBoundSequenceFromConcatenatedIterations(LateBoundSequenceFromConcatenatedIterationsContext ctx) {
+      this.stack.popStackFrame();
+    }
+
     // #endregion Scope management --------------------------------------------
 
   }

src/main/java/eu/europa/ted/efx/xpath/XPathScriptGenerator.java

@@ -257,6 +257,13 @@ public <T extends SequenceExpression> T composeForExpression(
         targetListType);
   }
 
+  @Override
+  public <T extends SequenceExpression> T composeForExpression(
+      IteratorListExpression iterators, SequenceExpression expression, Class<T> targetListType) {
+    return Expression.instantiate("for " + iterators.getScript() + " return " + expression.getScript(),
+        targetListType);
+  }
+
   @Override
   public IteratorExpression composeIteratorExpression(Expression variableDeclarationExpression, SequenceExpression sourceList) {
     return new IteratorExpression(variableDeclarationExpression.getScript() + " in " + sourceList.getScript());

src/test/java/eu/europa/ted/efx/sdk2/EfxExpressionTranslatorV2Test.java

@@ -1041,6 +1041,41 @@ void testDurationsFromDurationIteration_UsingFieldReference() {
         "ND-Root", "P1D in (for measure:$x in BT-00-Measure return P1D)");
   }
 
+  // Strings from concatenated iterations -----------------------------------
+
+  @Test
+  void testStringsFromConcatenatedIterations_UsingLiterals() {
+    testExpressionTranslationWithContext(
+        "'a' = (for $x in ('a','b','c') return ('x','y'))", "ND-Root",
+        "'a' in (for text:$x in ['a', 'b', 'c'] return ['x', 'y'])");
+  }
+
+  @Test
+  void testStringsFromConcatenatedIterations_UsingFieldReference() {
+    testExpressionTranslationWithContext(
+        "for $x in PathNode/TextField/normalize-space(text()) return PathNode/RepeatableTextField/normalize-space(text())",
+        "ND-Root",
+        "for text:$x in BT-00-Text return BT-00-Repeatable-Text");
+  }
+
+  // Return distinct (scalar) ------------------------------------------------
+
+  @Test
+  void testStringsFromIteration_ReturnDistinct() {
+    testExpressionTranslationWithContext(
+        "distinct-values(for $x in ('a','b','c') return concat($x, '!'))", "ND-Root",
+        "for text:$x in ['a', 'b', 'c'] return distinct concat($x, '!')");
+  }
+
+  // Return distinct (concatenated iterations / flatMap) --------------------
+
+  @Test
+  void testStringsFromConcatenatedIterations_ReturnDistinct() {
+    testExpressionTranslationWithContext(
+        "'a' = (distinct-values(for $x in ('a','b','c') return ('x','y')))", "ND-Root",
+        "'a' in (for text:$x in ['a', 'b', 'c'] return distinct ['x', 'y'])");
+  }
+
   // #endregion: Iteration expressions
 
   // #region: Numeric expressions ---------------------------------------------