[AURON #2173] [Correctness Testing] Add function-related tests

auron-spark-tests/common/src/test/scala/org/apache/spark/sql/SparkExpressionTestsBase.scala

@@ -35,6 +35,8 @@ import org.apache.spark.sql.types._
 import org.apache.spark.unsafe.types.UTF8String
 import org.scalactic.TripleEqualsSupport.Spread
 
+import org.apache.auron.sparkver
+
 /**
  * Base trait for all Spark expression tests.
  */
@@ -143,7 +145,7 @@ trait SparkExpressionTestsBase
       val empData = Seq(Row(1))
       _spark.createDataFrame(_spark.sparkContext.parallelize(empData), schema)
     }
-    val resultDF = df.select(Column(expression))
+    val resultDF = df.select(columnFromExpression(expression))
     val result = resultDF.collect()
 
     if (checkDataTypeSupported(expression) &&
@@ -296,23 +298,18 @@ trait SparkExpressionTestsBase
    */
   private def canConvertToDataFrame(inputRow: InternalRow): Boolean = {
     if (inputRow == EmptyRow || inputRow == InternalRow.empty) {
-      return true
-    }
-
-    if (!inputRow.isInstanceOf[GenericInternalRow]) {
-      return false
-    }
-
-    val values = inputRow.asInstanceOf[GenericInternalRow].values
-    for (value <- values) {
-      value match {
-        case _: MapData => return false
-        case _: ArrayData => return false
-        case _: InternalRow => return false
-        case _ =>
+      true
+    } else if (!inputRow.isInstanceOf[GenericInternalRow]) {
+      false
+    } else {
+      val values = inputRow.asInstanceOf[GenericInternalRow].values
+      values.forall {
+        case _: MapData => false
+        case _: ArrayData => false
+        case _: InternalRow => false
+        case _ => true
       }
     }
-    true
   }
 
   private def convertInternalRowToDataFrame(inputRow: InternalRow): DataFrame = {
@@ -350,8 +347,34 @@ trait SparkExpressionTestsBase
         structFieldSeq.append(StructField("n", IntegerType, nullable = true))
     }
 
-    _spark.internalCreateDataFrame(
+    internalCreateDataFrame(
       _spark.sparkContext.parallelize(Seq(inputRow)),
       StructType(structFieldSeq.toSeq))
   }
+
+  @sparkver("3.0 / 3.1 / 3.2 / 3.3 / 3.4 / 3.5")
+  private def columnFromExpression(expression: Expression): Column = {
+    Column(expression)
+  }
+
+  @sparkver("4.0 / 4.1")
+  private def columnFromExpression(expression: Expression): Column = {
+    new Column(org.apache.spark.sql.classic.ExpressionColumnNode(expression))
+  }
+
+  @sparkver("3.0 / 3.1 / 3.2 / 3.3 / 3.4 / 3.5")
+  private def internalCreateDataFrame(
+      rows: org.apache.spark.rdd.RDD[InternalRow],
+      schema: StructType): DataFrame = {
+    _spark.internalCreateDataFrame(rows, schema)
+  }
+
+  @sparkver("4.0 / 4.1")
+  private def internalCreateDataFrame(
+      rows: org.apache.spark.rdd.RDD[InternalRow],
+      schema: StructType): org.apache.spark.sql.classic.DataFrame = {
+    _spark
+      .asInstanceOf[org.apache.spark.sql.classic.SparkSession]
+      .internalCreateDataFrame(rows, schema)
+  }
 }

auron-spark-tests/common/src/test/scala/org/apache/spark/sql/SparkQueryTestsBase.scala

@@ -91,9 +91,9 @@ object AuronQueryTestUtil extends Assertions {
       df: DataFrame,
       expectedAnswer: Seq[Row],
       checkToRDD: Boolean = true): Option[String] = {
-    val isSorted = df.logicalPlan.collect { case s: logical.Sort => s }.nonEmpty
+    val isSorted = df.queryExecution.logical.collect { case s: logical.Sort => s }.nonEmpty
     if (checkToRDD) {
-      SQLExecution.withSQLConfPropagated(df.sparkSession) {
+      SQLExecution.withSQLConfPropagated(df.queryExecution.sparkSession) {
         df.rdd.count() // Also attempt to deserialize as an RDD [SPARK-15791]
       }
     }

auron-spark-tests/spark31/src/test/scala/org/apache/auron/utils/AuronSparkTestSettings.scala

@@ -16,7 +16,57 @@
  */
 package org.apache.auron.utils
 
+import org.apache.spark.sql._
+
 class AuronSparkTestSettings extends SparkTestSettings {
+  {
+    // Use Arrow's unsafe implementation.
+    System.setProperty("arrow.allocation.manager.type", "Unsafe")
+  }
+
+
+
+  enableSuite[AuronDataFrameFunctionsSuite]
+    // Native execution wraps SparkRuntimeException from map construction in SparkException.
+    .exclude("map with arrays")
+    // Native execution wraps SparkRuntimeException from map_concat input validation in
+    // SparkException.
+    .exclude("map_concat function")
+    // Native reverse only supports string inputs; array inputs fail with unsupported List type.
+    .exclude("reverse function - array for primitive type not containing null")
+    .exclude("reverse function - array for primitive type containing null")
+    .exclude("reverse function - array for non-primitive type")
+    // Native flatten can fail when child arrays have different containsNull metadata.
+    .exclude("flatten function")
+    // Native execution wraps SparkRuntimeException from null map keys in SparkException.
+    .exclude("SPARK-24734: Fix containsNull of Concat for array type")
+
+  enableSuite[AuronDateFunctionsSuite]
+    // Native execution wraps SparkUpgradeException from parsing validation in SparkException.
+    .exclude("function to_date")
+    // Native date_trunc does not support all Spark granularity aliases.
+    .exclude("function date_trunc")
+    // Native date_trunc throws for unsupported fields instead of returning NULL as Spark does.
+    .exclude("unsupported fmt fields for trunc/date_trunc results null")
+    // Native execution wraps SparkUpgradeException from parsing validation in SparkException.
+    .exclude("unix_timestamp")
+    // Native execution wraps IllegalArgumentException from format validation in SparkException.
+    .exclude("to_unix_timestamp")
+    // Native date_trunc may produce incorrect results for historical timestamps with
+    // non-UTC timezones due to timezone handling differences in the DataFusion engine.
+    .exclude("SPARK-30766: date_trunc of old timestamps to hours and days")
+
+  enableSuite[AuronMathFunctionsSuite]
+    // Native acosh uses a different floating-point formula than Spark's StrictMath.log,
+    // producing results that differ at the last ULP for certain edge-case inputs.
+    .exclude("acosh")
+
+  enableSuite[AuronMiscFunctionsSuite]
+
+  enableSuite[AuronStringFunctionsSuite]
+    // See https://github.com/apache/auron/issues/1724
+    .exclude("string / binary substring function")
+
   override def getSQLQueryTestSettings: SQLQueryTestSettings = new SQLQueryTestSettings {
     override def getResourceFilePath: String = ""
     override def getSupportedSQLQueryTests: Set[String] = Set.empty

auron-spark-tests/spark31/src/test/scala/org/apache/spark/sql/AuronDataFrameFunctionsSuite.scala

@@ -0,0 +1,19 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.spark.sql
+
+class AuronDataFrameFunctionsSuite extends DataFrameFunctionsSuite with SparkQueryTestsBase {}

auron-spark-tests/spark31/src/test/scala/org/apache/spark/sql/AuronDateFunctionsSuite.scala

@@ -0,0 +1,19 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.spark.sql
+
+class AuronDateFunctionsSuite extends DateFunctionsSuite with SparkQueryTestsBase {}

auron-spark-tests/spark31/src/test/scala/org/apache/spark/sql/AuronMathFunctionsSuite.scala

@@ -0,0 +1,19 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.spark.sql
+
+class AuronMathFunctionsSuite extends MathFunctionsSuite with SparkQueryTestsBase {}

auron-spark-tests/spark31/src/test/scala/org/apache/spark/sql/AuronMiscFunctionsSuite.scala

@@ -0,0 +1,19 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.spark.sql
+
+class AuronMiscFunctionsSuite extends MiscFunctionsSuite with SparkQueryTestsBase {}

auron-spark-tests/spark31/src/test/scala/org/apache/spark/sql/AuronStringFunctionsSuite.scala

@@ -0,0 +1,19 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.spark.sql
+
+class AuronStringFunctionsSuite extends StringFunctionsSuite with SparkQueryTestsBase {}

auron-spark-tests/spark32/src/test/scala/org/apache/auron/utils/AuronSparkTestSettings.scala

@@ -16,7 +16,52 @@
  */
 package org.apache.auron.utils
 
+import org.apache.spark.sql._
+
 class AuronSparkTestSettings extends SparkTestSettings {
+  {
+    // Use Arrow's unsafe implementation.
+    System.setProperty("arrow.allocation.manager.type", "Unsafe")
+  }
+
+  enableSuite[AuronDataFrameFunctionsSuite]
+    // Native execution wraps SparkRuntimeException from map-related validation in
+    // SparkException.
+    .exclude("map with arrays")
+    .exclude("map_concat function")
+    .exclude("SPARK-24734: Fix containsNull of Concat for array type")
+    // Native reverse only supports string inputs; array inputs fail with unsupported List type.
+    .exclude("reverse function - array for primitive type not containing null")
+    .exclude("reverse function - array for primitive type containing null")
+    .exclude("reverse function - array for non-primitive type")
+    // Native flatten can fail when child arrays have different containsNull metadata.
+    .exclude("flatten function")
+
+  enableSuite[AuronDateFunctionsSuite]
+    // Native execution wraps Spark parsing/format validation exceptions in SparkException.
+    .exclude("function to_date")
+    .exclude("unix_timestamp")
+    .exclude("to_unix_timestamp")
+    // Native date_trunc does not support all Spark granularity aliases.
+    .exclude("function date_trunc")
+    // Native date_trunc throws for unsupported fields instead of returning NULL as Spark does.
+    .exclude("unsupported fmt fields for trunc/date_trunc results null")
+    // Native date_trunc may produce incorrect results for historical timestamps with
+    // non-UTC timezones due to timezone handling differences in the DataFusion engine.
+    .exclude("SPARK-30766: date_trunc of old timestamps to hours and days")
+
+  enableSuite[AuronMathFunctionsSuite]
+    // Native acosh uses a different floating-point formula than Spark's StrictMath.log,
+    // producing results that differ at the last ULP for certain edge-case inputs.
+    .exclude("acosh")
+
+  enableSuite[AuronMiscFunctionsSuite]
+
+  enableSuite[AuronStringFunctionsSuite]
+    // Native substr does not support BinaryType inputs.
+    // See https://github.com/apache/auron/issues/1724
+    .exclude("string / binary substring function")
+
   override def getSQLQueryTestSettings: SQLQueryTestSettings = new SQLQueryTestSettings {
     override def getResourceFilePath: String = ""
     override def getSupportedSQLQueryTests: Set[String] = Set.empty

auron-spark-tests/spark32/src/test/scala/org/apache/spark/sql/AuronDataFrameFunctionsSuite.scala

@@ -0,0 +1,19 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.spark.sql
+
+class AuronDataFrameFunctionsSuite extends DataFrameFunctionsSuite with SparkQueryTestsBase {}

auron-spark-tests/spark32/src/test/scala/org/apache/spark/sql/AuronDateFunctionsSuite.scala

@@ -0,0 +1,19 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.spark.sql
+
+class AuronDateFunctionsSuite extends DateFunctionsSuite with SparkQueryTestsBase {}

auron-spark-tests/spark32/src/test/scala/org/apache/spark/sql/AuronMathFunctionsSuite.scala

@@ -0,0 +1,19 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.spark.sql
+
+class AuronMathFunctionsSuite extends MathFunctionsSuite with SparkQueryTestsBase {}

auron-spark-tests/spark32/src/test/scala/org/apache/spark/sql/AuronMiscFunctionsSuite.scala

@@ -0,0 +1,19 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.spark.sql
+
+class AuronMiscFunctionsSuite extends MiscFunctionsSuite with SparkQueryTestsBase {}