diff --git a/auron-spark-tests/common/src/test/scala/org/apache/spark/sql/SparkExpressionTestsBase.scala b/auron-spark-tests/common/src/test/scala/org/apache/spark/sql/SparkExpressionTestsBase.scala index 14490aef0..041607656 100644 --- a/auron-spark-tests/common/src/test/scala/org/apache/spark/sql/SparkExpressionTestsBase.scala +++ b/auron-spark-tests/common/src/test/scala/org/apache/spark/sql/SparkExpressionTestsBase.scala @@ -35,6 +35,8 @@ import org.apache.spark.sql.types._ import org.apache.spark.unsafe.types.UTF8String import org.scalactic.TripleEqualsSupport.Spread +import org.apache.auron.sparkver + /** * Base trait for all Spark expression tests. */ @@ -143,7 +145,7 @@ trait SparkExpressionTestsBase val empData = Seq(Row(1)) _spark.createDataFrame(_spark.sparkContext.parallelize(empData), schema) } - val resultDF = df.select(Column(expression)) + val resultDF = df.select(columnFromExpression(expression)) val result = resultDF.collect() if (checkDataTypeSupported(expression) && @@ -296,23 +298,18 @@ trait SparkExpressionTestsBase */ private def canConvertToDataFrame(inputRow: InternalRow): Boolean = { if (inputRow == EmptyRow || inputRow == InternalRow.empty) { - return true - } - - if (!inputRow.isInstanceOf[GenericInternalRow]) { - return false - } - - val values = inputRow.asInstanceOf[GenericInternalRow].values - for (value <- values) { - value match { - case _: MapData => return false - case _: ArrayData => return false - case _: InternalRow => return false - case _ => + true + } else if (!inputRow.isInstanceOf[GenericInternalRow]) { + false + } else { + val values = inputRow.asInstanceOf[GenericInternalRow].values + values.forall { + case _: MapData => false + case _: ArrayData => false + case _: InternalRow => false + case _ => true } } - true } private def convertInternalRowToDataFrame(inputRow: InternalRow): DataFrame = { @@ -350,8 +347,34 @@ trait SparkExpressionTestsBase structFieldSeq.append(StructField("n", IntegerType, nullable = true)) } - _spark.internalCreateDataFrame( + internalCreateDataFrame( _spark.sparkContext.parallelize(Seq(inputRow)), StructType(structFieldSeq.toSeq)) } + + @sparkver("3.0 / 3.1 / 3.2 / 3.3 / 3.4 / 3.5") + private def columnFromExpression(expression: Expression): Column = { + Column(expression) + } + + @sparkver("4.0 / 4.1") + private def columnFromExpression(expression: Expression): Column = { + new Column(org.apache.spark.sql.classic.ExpressionColumnNode(expression)) + } + + @sparkver("3.0 / 3.1 / 3.2 / 3.3 / 3.4 / 3.5") + private def internalCreateDataFrame( + rows: org.apache.spark.rdd.RDD[InternalRow], + schema: StructType): DataFrame = { + _spark.internalCreateDataFrame(rows, schema) + } + + @sparkver("4.0 / 4.1") + private def internalCreateDataFrame( + rows: org.apache.spark.rdd.RDD[InternalRow], + schema: StructType): org.apache.spark.sql.classic.DataFrame = { + _spark + .asInstanceOf[org.apache.spark.sql.classic.SparkSession] + .internalCreateDataFrame(rows, schema) + } } diff --git a/auron-spark-tests/common/src/test/scala/org/apache/spark/sql/SparkQueryTestsBase.scala b/auron-spark-tests/common/src/test/scala/org/apache/spark/sql/SparkQueryTestsBase.scala index 159cb6174..8b3db0256 100644 --- a/auron-spark-tests/common/src/test/scala/org/apache/spark/sql/SparkQueryTestsBase.scala +++ b/auron-spark-tests/common/src/test/scala/org/apache/spark/sql/SparkQueryTestsBase.scala @@ -91,9 +91,9 @@ object AuronQueryTestUtil extends Assertions { df: DataFrame, expectedAnswer: Seq[Row], checkToRDD: Boolean = true): Option[String] = { - val isSorted = df.logicalPlan.collect { case s: logical.Sort => s }.nonEmpty + val isSorted = df.queryExecution.logical.collect { case s: logical.Sort => s }.nonEmpty if (checkToRDD) { - SQLExecution.withSQLConfPropagated(df.sparkSession) { + SQLExecution.withSQLConfPropagated(df.queryExecution.sparkSession) { df.rdd.count() // Also attempt to deserialize as an RDD [SPARK-15791] } } diff --git a/auron-spark-tests/spark31/src/test/scala/org/apache/auron/utils/AuronSparkTestSettings.scala b/auron-spark-tests/spark31/src/test/scala/org/apache/auron/utils/AuronSparkTestSettings.scala index 052cca5d1..331724bf7 100644 --- a/auron-spark-tests/spark31/src/test/scala/org/apache/auron/utils/AuronSparkTestSettings.scala +++ b/auron-spark-tests/spark31/src/test/scala/org/apache/auron/utils/AuronSparkTestSettings.scala @@ -16,7 +16,57 @@ */ package org.apache.auron.utils +import org.apache.spark.sql._ + class AuronSparkTestSettings extends SparkTestSettings { + { + // Use Arrow's unsafe implementation. + System.setProperty("arrow.allocation.manager.type", "Unsafe") + } + + + + enableSuite[AuronDataFrameFunctionsSuite] + // Native execution wraps SparkRuntimeException from map construction in SparkException. + .exclude("map with arrays") + // Native execution wraps SparkRuntimeException from map_concat input validation in + // SparkException. + .exclude("map_concat function") + // Native reverse only supports string inputs; array inputs fail with unsupported List type. + .exclude("reverse function - array for primitive type not containing null") + .exclude("reverse function - array for primitive type containing null") + .exclude("reverse function - array for non-primitive type") + // Native flatten can fail when child arrays have different containsNull metadata. + .exclude("flatten function") + // Native execution wraps SparkRuntimeException from null map keys in SparkException. + .exclude("SPARK-24734: Fix containsNull of Concat for array type") + + enableSuite[AuronDateFunctionsSuite] + // Native execution wraps SparkUpgradeException from parsing validation in SparkException. + .exclude("function to_date") + // Native date_trunc does not support all Spark granularity aliases. + .exclude("function date_trunc") + // Native date_trunc throws for unsupported fields instead of returning NULL as Spark does. + .exclude("unsupported fmt fields for trunc/date_trunc results null") + // Native execution wraps SparkUpgradeException from parsing validation in SparkException. + .exclude("unix_timestamp") + // Native execution wraps IllegalArgumentException from format validation in SparkException. + .exclude("to_unix_timestamp") + // Native date_trunc may produce incorrect results for historical timestamps with + // non-UTC timezones due to timezone handling differences in the DataFusion engine. + .exclude("SPARK-30766: date_trunc of old timestamps to hours and days") + + enableSuite[AuronMathFunctionsSuite] + // Native acosh uses a different floating-point formula than Spark's StrictMath.log, + // producing results that differ at the last ULP for certain edge-case inputs. + .exclude("acosh") + + enableSuite[AuronMiscFunctionsSuite] + + enableSuite[AuronStringFunctionsSuite] + // See https://github.com/apache/auron/issues/1724 + .exclude("string / binary substring function") + override def getSQLQueryTestSettings: SQLQueryTestSettings = new SQLQueryTestSettings { override def getResourceFilePath: String = "" override def getSupportedSQLQueryTests: Set[String] = Set.empty diff --git a/auron-spark-tests/spark31/src/test/scala/org/apache/spark/sql/AuronDataFrameFunctionsSuite.scala b/auron-spark-tests/spark31/src/test/scala/org/apache/spark/sql/AuronDataFrameFunctionsSuite.scala new file mode 100644 index 000000000..98448e333 --- /dev/null +++ b/auron-spark-tests/spark31/src/test/scala/org/apache/spark/sql/AuronDataFrameFunctionsSuite.scala @@ -0,0 +1,19 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql + +class AuronDataFrameFunctionsSuite extends DataFrameFunctionsSuite with SparkQueryTestsBase {} diff --git a/auron-spark-tests/spark31/src/test/scala/org/apache/spark/sql/AuronDateFunctionsSuite.scala b/auron-spark-tests/spark31/src/test/scala/org/apache/spark/sql/AuronDateFunctionsSuite.scala new file mode 100644 index 000000000..dddef76fb --- /dev/null +++ b/auron-spark-tests/spark31/src/test/scala/org/apache/spark/sql/AuronDateFunctionsSuite.scala @@ -0,0 +1,19 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql + +class AuronDateFunctionsSuite extends DateFunctionsSuite with SparkQueryTestsBase {} diff --git a/auron-spark-tests/spark31/src/test/scala/org/apache/spark/sql/AuronMathFunctionsSuite.scala b/auron-spark-tests/spark31/src/test/scala/org/apache/spark/sql/AuronMathFunctionsSuite.scala new file mode 100644 index 000000000..eb2d96a9b --- /dev/null +++ b/auron-spark-tests/spark31/src/test/scala/org/apache/spark/sql/AuronMathFunctionsSuite.scala @@ -0,0 +1,19 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql + +class AuronMathFunctionsSuite extends MathFunctionsSuite with SparkQueryTestsBase {} diff --git a/auron-spark-tests/spark31/src/test/scala/org/apache/spark/sql/AuronMiscFunctionsSuite.scala b/auron-spark-tests/spark31/src/test/scala/org/apache/spark/sql/AuronMiscFunctionsSuite.scala new file mode 100644 index 000000000..a50ff3367 --- /dev/null +++ b/auron-spark-tests/spark31/src/test/scala/org/apache/spark/sql/AuronMiscFunctionsSuite.scala @@ -0,0 +1,19 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql + +class AuronMiscFunctionsSuite extends MiscFunctionsSuite with SparkQueryTestsBase {} diff --git a/auron-spark-tests/spark31/src/test/scala/org/apache/spark/sql/AuronStringFunctionsSuite.scala b/auron-spark-tests/spark31/src/test/scala/org/apache/spark/sql/AuronStringFunctionsSuite.scala new file mode 100644 index 000000000..bd621ba17 --- /dev/null +++ b/auron-spark-tests/spark31/src/test/scala/org/apache/spark/sql/AuronStringFunctionsSuite.scala @@ -0,0 +1,19 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql + +class AuronStringFunctionsSuite extends StringFunctionsSuite with SparkQueryTestsBase {} diff --git a/auron-spark-tests/spark32/src/test/scala/org/apache/auron/utils/AuronSparkTestSettings.scala b/auron-spark-tests/spark32/src/test/scala/org/apache/auron/utils/AuronSparkTestSettings.scala index 052cca5d1..cca7adc88 100644 --- a/auron-spark-tests/spark32/src/test/scala/org/apache/auron/utils/AuronSparkTestSettings.scala +++ b/auron-spark-tests/spark32/src/test/scala/org/apache/auron/utils/AuronSparkTestSettings.scala @@ -16,7 +16,52 @@ */ package org.apache.auron.utils +import org.apache.spark.sql._ + class AuronSparkTestSettings extends SparkTestSettings { + { + // Use Arrow's unsafe implementation. + System.setProperty("arrow.allocation.manager.type", "Unsafe") + } + + enableSuite[AuronDataFrameFunctionsSuite] + // Native execution wraps SparkRuntimeException from map-related validation in + // SparkException. + .exclude("map with arrays") + .exclude("map_concat function") + .exclude("SPARK-24734: Fix containsNull of Concat for array type") + // Native reverse only supports string inputs; array inputs fail with unsupported List type. + .exclude("reverse function - array for primitive type not containing null") + .exclude("reverse function - array for primitive type containing null") + .exclude("reverse function - array for non-primitive type") + // Native flatten can fail when child arrays have different containsNull metadata. + .exclude("flatten function") + + enableSuite[AuronDateFunctionsSuite] + // Native execution wraps Spark parsing/format validation exceptions in SparkException. + .exclude("function to_date") + .exclude("unix_timestamp") + .exclude("to_unix_timestamp") + // Native date_trunc does not support all Spark granularity aliases. + .exclude("function date_trunc") + // Native date_trunc throws for unsupported fields instead of returning NULL as Spark does. + .exclude("unsupported fmt fields for trunc/date_trunc results null") + // Native date_trunc may produce incorrect results for historical timestamps with + // non-UTC timezones due to timezone handling differences in the DataFusion engine. + .exclude("SPARK-30766: date_trunc of old timestamps to hours and days") + + enableSuite[AuronMathFunctionsSuite] + // Native acosh uses a different floating-point formula than Spark's StrictMath.log, + // producing results that differ at the last ULP for certain edge-case inputs. + .exclude("acosh") + + enableSuite[AuronMiscFunctionsSuite] + + enableSuite[AuronStringFunctionsSuite] + // Native substr does not support BinaryType inputs. + // See https://github.com/apache/auron/issues/1724 + .exclude("string / binary substring function") + override def getSQLQueryTestSettings: SQLQueryTestSettings = new SQLQueryTestSettings { override def getResourceFilePath: String = "" override def getSupportedSQLQueryTests: Set[String] = Set.empty diff --git a/auron-spark-tests/spark32/src/test/scala/org/apache/spark/sql/AuronDataFrameFunctionsSuite.scala b/auron-spark-tests/spark32/src/test/scala/org/apache/spark/sql/AuronDataFrameFunctionsSuite.scala new file mode 100644 index 000000000..98448e333 --- /dev/null +++ b/auron-spark-tests/spark32/src/test/scala/org/apache/spark/sql/AuronDataFrameFunctionsSuite.scala @@ -0,0 +1,19 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql + +class AuronDataFrameFunctionsSuite extends DataFrameFunctionsSuite with SparkQueryTestsBase {} diff --git a/auron-spark-tests/spark32/src/test/scala/org/apache/spark/sql/AuronDateFunctionsSuite.scala b/auron-spark-tests/spark32/src/test/scala/org/apache/spark/sql/AuronDateFunctionsSuite.scala new file mode 100644 index 000000000..dddef76fb --- /dev/null +++ b/auron-spark-tests/spark32/src/test/scala/org/apache/spark/sql/AuronDateFunctionsSuite.scala @@ -0,0 +1,19 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql + +class AuronDateFunctionsSuite extends DateFunctionsSuite with SparkQueryTestsBase {} diff --git a/auron-spark-tests/spark32/src/test/scala/org/apache/spark/sql/AuronMathFunctionsSuite.scala b/auron-spark-tests/spark32/src/test/scala/org/apache/spark/sql/AuronMathFunctionsSuite.scala new file mode 100644 index 000000000..eb2d96a9b --- /dev/null +++ b/auron-spark-tests/spark32/src/test/scala/org/apache/spark/sql/AuronMathFunctionsSuite.scala @@ -0,0 +1,19 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql + +class AuronMathFunctionsSuite extends MathFunctionsSuite with SparkQueryTestsBase {} diff --git a/auron-spark-tests/spark32/src/test/scala/org/apache/spark/sql/AuronMiscFunctionsSuite.scala b/auron-spark-tests/spark32/src/test/scala/org/apache/spark/sql/AuronMiscFunctionsSuite.scala new file mode 100644 index 000000000..a50ff3367 --- /dev/null +++ b/auron-spark-tests/spark32/src/test/scala/org/apache/spark/sql/AuronMiscFunctionsSuite.scala @@ -0,0 +1,19 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql + +class AuronMiscFunctionsSuite extends MiscFunctionsSuite with SparkQueryTestsBase {} diff --git a/auron-spark-tests/spark32/src/test/scala/org/apache/spark/sql/AuronStringFunctionsSuite.scala b/auron-spark-tests/spark32/src/test/scala/org/apache/spark/sql/AuronStringFunctionsSuite.scala new file mode 100644 index 000000000..bd621ba17 --- /dev/null +++ b/auron-spark-tests/spark32/src/test/scala/org/apache/spark/sql/AuronStringFunctionsSuite.scala @@ -0,0 +1,19 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql + +class AuronStringFunctionsSuite extends StringFunctionsSuite with SparkQueryTestsBase {} diff --git a/auron-spark-tests/spark34/src/test/scala/org/apache/auron/utils/AuronSparkTestSettings.scala b/auron-spark-tests/spark34/src/test/scala/org/apache/auron/utils/AuronSparkTestSettings.scala index 052cca5d1..150fe7b4b 100644 --- a/auron-spark-tests/spark34/src/test/scala/org/apache/auron/utils/AuronSparkTestSettings.scala +++ b/auron-spark-tests/spark34/src/test/scala/org/apache/auron/utils/AuronSparkTestSettings.scala @@ -16,7 +16,53 @@ */ package org.apache.auron.utils +import org.apache.spark.sql._ + class AuronSparkTestSettings extends SparkTestSettings { + { + // Use Arrow's unsafe implementation. + System.setProperty("arrow.allocation.manager.type", "Unsafe") + } + + enableSuite[AuronDataFrameFunctionsSuite] + // Native execution wraps SparkRuntimeException from validation failures in SparkException. + .exclude("map with arrays") + .exclude("map_concat function") + .exclude("SPARK-24734: Fix containsNull of Concat for array type") + .exclude("array_insert functions") + .exclude("transform keys function - Invalid lambda functions and exceptions") + // Native reverse only supports string inputs; array inputs fail with unsupported List type. + .exclude("reverse function - array for primitive type not containing null") + .exclude("reverse function - array for primitive type containing null") + .exclude("reverse function - array for non-primitive type") + // Native flatten can fail when child arrays have different containsNull metadata. + .exclude("flatten function") + + enableSuite[AuronDateFunctionsSuite] + // Native execution wraps Spark parsing/format validation exceptions in SparkException. + .exclude("function to_date") + .exclude("unix_timestamp") + .exclude("to_unix_timestamp") + // Native date_trunc does not support all Spark granularity aliases. + .exclude("function date_trunc") + // Native date_trunc throws for unsupported fields instead of returning NULL as Spark does. + .exclude("unsupported fmt fields for trunc/date_trunc results null") + // Native date_trunc may produce incorrect results for historical timestamps with + // non-UTC timezones due to timezone handling differences in the DataFusion engine. + .exclude("SPARK-30766: date_trunc of old timestamps to hours and days") + + enableSuite[AuronMathFunctionsSuite] + // Native acosh uses a different floating-point formula than Spark's StrictMath.log, + // producing results that differ at the last ULP for certain edge-case inputs. + .exclude("acosh") + + enableSuite[AuronMiscFunctionsSuite] + + enableSuite[AuronStringFunctionsSuite] + // Native substr does not support BinaryType inputs. + // See https://github.com/apache/auron/issues/1724 + .exclude("string / binary substring function") + override def getSQLQueryTestSettings: SQLQueryTestSettings = new SQLQueryTestSettings { override def getResourceFilePath: String = "" override def getSupportedSQLQueryTests: Set[String] = Set.empty diff --git a/auron-spark-tests/spark34/src/test/scala/org/apache/spark/sql/AuronDataFrameFunctionsSuite.scala b/auron-spark-tests/spark34/src/test/scala/org/apache/spark/sql/AuronDataFrameFunctionsSuite.scala new file mode 100644 index 000000000..98448e333 --- /dev/null +++ b/auron-spark-tests/spark34/src/test/scala/org/apache/spark/sql/AuronDataFrameFunctionsSuite.scala @@ -0,0 +1,19 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql + +class AuronDataFrameFunctionsSuite extends DataFrameFunctionsSuite with SparkQueryTestsBase {} diff --git a/auron-spark-tests/spark34/src/test/scala/org/apache/spark/sql/AuronDateFunctionsSuite.scala b/auron-spark-tests/spark34/src/test/scala/org/apache/spark/sql/AuronDateFunctionsSuite.scala new file mode 100644 index 000000000..dddef76fb --- /dev/null +++ b/auron-spark-tests/spark34/src/test/scala/org/apache/spark/sql/AuronDateFunctionsSuite.scala @@ -0,0 +1,19 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql + +class AuronDateFunctionsSuite extends DateFunctionsSuite with SparkQueryTestsBase {} diff --git a/auron-spark-tests/spark34/src/test/scala/org/apache/spark/sql/AuronMathFunctionsSuite.scala b/auron-spark-tests/spark34/src/test/scala/org/apache/spark/sql/AuronMathFunctionsSuite.scala new file mode 100644 index 000000000..eb2d96a9b --- /dev/null +++ b/auron-spark-tests/spark34/src/test/scala/org/apache/spark/sql/AuronMathFunctionsSuite.scala @@ -0,0 +1,19 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql + +class AuronMathFunctionsSuite extends MathFunctionsSuite with SparkQueryTestsBase {} diff --git a/auron-spark-tests/spark34/src/test/scala/org/apache/spark/sql/AuronMiscFunctionsSuite.scala b/auron-spark-tests/spark34/src/test/scala/org/apache/spark/sql/AuronMiscFunctionsSuite.scala new file mode 100644 index 000000000..a50ff3367 --- /dev/null +++ b/auron-spark-tests/spark34/src/test/scala/org/apache/spark/sql/AuronMiscFunctionsSuite.scala @@ -0,0 +1,19 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql + +class AuronMiscFunctionsSuite extends MiscFunctionsSuite with SparkQueryTestsBase {} diff --git a/auron-spark-tests/spark34/src/test/scala/org/apache/spark/sql/AuronStringFunctionsSuite.scala b/auron-spark-tests/spark34/src/test/scala/org/apache/spark/sql/AuronStringFunctionsSuite.scala new file mode 100644 index 000000000..bd621ba17 --- /dev/null +++ b/auron-spark-tests/spark34/src/test/scala/org/apache/spark/sql/AuronStringFunctionsSuite.scala @@ -0,0 +1,19 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql + +class AuronStringFunctionsSuite extends StringFunctionsSuite with SparkQueryTestsBase {} diff --git a/auron-spark-tests/spark35/src/test/scala/org/apache/auron/utils/AuronSparkTestSettings.scala b/auron-spark-tests/spark35/src/test/scala/org/apache/auron/utils/AuronSparkTestSettings.scala index 052cca5d1..697e314a4 100644 --- a/auron-spark-tests/spark35/src/test/scala/org/apache/auron/utils/AuronSparkTestSettings.scala +++ b/auron-spark-tests/spark35/src/test/scala/org/apache/auron/utils/AuronSparkTestSettings.scala @@ -16,7 +16,55 @@ */ package org.apache.auron.utils +import org.apache.spark.sql._ + class AuronSparkTestSettings extends SparkTestSettings { + { + // Use Arrow's unsafe implementation. + System.setProperty("arrow.allocation.manager.type", "Unsafe") + } + + enableSuite[AuronDataFrameFunctionsSuite] + // Native execution wraps SparkRuntimeException from validation failures in SparkException. + .exclude("map with arrays") + .exclude("map_concat function") + .exclude("SPARK-24734: Fix containsNull of Concat for array type") + .exclude("array_insert functions") + .exclude("transform keys function - Invalid lambda functions and exceptions") + // Native reverse only supports string inputs; array inputs fail with unsupported List type. + .exclude("reverse function - array for primitive type not containing null") + .exclude("reverse function - array for primitive type containing null") + .exclude("reverse function - array for non-primitive type") + // Native flatten can fail when child arrays have different containsNull metadata. + .exclude("flatten function") + + enableSuite[AuronDateFunctionsSuite] + // Native execution wraps Spark parsing/format validation exceptions in SparkException. + .exclude("function to_date") + .exclude("unix_timestamp") + .exclude("to_unix_timestamp") + // Native date_trunc does not support all Spark granularity aliases. + .exclude("function date_trunc") + // Native date_trunc throws for unsupported fields instead of returning NULL as Spark does. + .exclude("unsupported fmt fields for trunc/date_trunc results null") + // Native date_trunc may produce incorrect results for historical timestamps with + // non-UTC timezones due to timezone handling differences in the DataFusion engine. + .exclude("SPARK-30766: date_trunc of old timestamps to hours and days") + + enableSuite[AuronMathFunctionsSuite] + // Native acosh uses a different floating-point formula than Spark's StrictMath.log, + // producing results that differ at the last ULP for certain edge-case inputs. + .exclude("acosh") + + enableSuite[AuronMiscFunctionsSuite] + + enableSuite[AuronStringFunctionsSuite] + // Native levenshtein has a Spark 3.5+ result or schema comparison mismatch. + .exclude("string Levenshtein distance") + // Native substr does not support BinaryType inputs. + // See https://github.com/apache/auron/issues/1724 + .exclude("string / binary substring function") + override def getSQLQueryTestSettings: SQLQueryTestSettings = new SQLQueryTestSettings { override def getResourceFilePath: String = "" override def getSupportedSQLQueryTests: Set[String] = Set.empty diff --git a/auron-spark-tests/spark35/src/test/scala/org/apache/spark/sql/AuronDataFrameFunctionsSuite.scala b/auron-spark-tests/spark35/src/test/scala/org/apache/spark/sql/AuronDataFrameFunctionsSuite.scala new file mode 100644 index 000000000..98448e333 --- /dev/null +++ b/auron-spark-tests/spark35/src/test/scala/org/apache/spark/sql/AuronDataFrameFunctionsSuite.scala @@ -0,0 +1,19 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql + +class AuronDataFrameFunctionsSuite extends DataFrameFunctionsSuite with SparkQueryTestsBase {} diff --git a/auron-spark-tests/spark35/src/test/scala/org/apache/spark/sql/AuronDateFunctionsSuite.scala b/auron-spark-tests/spark35/src/test/scala/org/apache/spark/sql/AuronDateFunctionsSuite.scala new file mode 100644 index 000000000..dddef76fb --- /dev/null +++ b/auron-spark-tests/spark35/src/test/scala/org/apache/spark/sql/AuronDateFunctionsSuite.scala @@ -0,0 +1,19 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql + +class AuronDateFunctionsSuite extends DateFunctionsSuite with SparkQueryTestsBase {} diff --git a/auron-spark-tests/spark35/src/test/scala/org/apache/spark/sql/AuronMathFunctionsSuite.scala b/auron-spark-tests/spark35/src/test/scala/org/apache/spark/sql/AuronMathFunctionsSuite.scala new file mode 100644 index 000000000..eb2d96a9b --- /dev/null +++ b/auron-spark-tests/spark35/src/test/scala/org/apache/spark/sql/AuronMathFunctionsSuite.scala @@ -0,0 +1,19 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql + +class AuronMathFunctionsSuite extends MathFunctionsSuite with SparkQueryTestsBase {} diff --git a/auron-spark-tests/spark35/src/test/scala/org/apache/spark/sql/AuronMiscFunctionsSuite.scala b/auron-spark-tests/spark35/src/test/scala/org/apache/spark/sql/AuronMiscFunctionsSuite.scala new file mode 100644 index 000000000..a50ff3367 --- /dev/null +++ b/auron-spark-tests/spark35/src/test/scala/org/apache/spark/sql/AuronMiscFunctionsSuite.scala @@ -0,0 +1,19 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql + +class AuronMiscFunctionsSuite extends MiscFunctionsSuite with SparkQueryTestsBase {} diff --git a/auron-spark-tests/spark35/src/test/scala/org/apache/spark/sql/AuronStringFunctionsSuite.scala b/auron-spark-tests/spark35/src/test/scala/org/apache/spark/sql/AuronStringFunctionsSuite.scala new file mode 100644 index 000000000..bd621ba17 --- /dev/null +++ b/auron-spark-tests/spark35/src/test/scala/org/apache/spark/sql/AuronStringFunctionsSuite.scala @@ -0,0 +1,19 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql + +class AuronStringFunctionsSuite extends StringFunctionsSuite with SparkQueryTestsBase {} diff --git a/auron-spark-tests/spark40/src/test/scala/org/apache/auron/utils/AuronSparkTestSettings.scala b/auron-spark-tests/spark40/src/test/scala/org/apache/auron/utils/AuronSparkTestSettings.scala index 052cca5d1..a04eaebe1 100644 --- a/auron-spark-tests/spark40/src/test/scala/org/apache/auron/utils/AuronSparkTestSettings.scala +++ b/auron-spark-tests/spark40/src/test/scala/org/apache/auron/utils/AuronSparkTestSettings.scala @@ -16,7 +16,55 @@ */ package org.apache.auron.utils +import org.apache.spark.sql._ + class AuronSparkTestSettings extends SparkTestSettings { + { + // Use Arrow's unsafe implementation. + System.setProperty("arrow.allocation.manager.type", "Unsafe") + } + + enableSuite[AuronDataFrameFunctionsSuite] + // Native execution wraps SparkRuntimeException from validation failures in SparkException. + .exclude("map with arrays") + .exclude("map_concat function") + .exclude("SPARK-24734: Fix containsNull of Concat for array type") + .exclude("array_insert functions") + .exclude("transform keys function - Invalid lambda functions and exceptions") + // Native reverse only supports string inputs; array inputs fail with unsupported List type. + .exclude("reverse function - array for primitive type not containing null") + .exclude("reverse function - array for primitive type containing null") + .exclude("reverse function - array for non-primitive type") + // Native flatten can panic when child arrays have different containsNull metadata. + .exclude("flatten function") + + enableSuite[AuronDateFunctionsSuite] + // Native execution wraps Spark parsing/format validation exceptions in SparkException. + .exclude("function to_date") + .exclude("unix_timestamp") + .exclude("to_unix_timestamp") + // Native date_trunc does not support all Spark aliases such as "yy". + .exclude("function date_trunc") + // Native date_trunc throws for unsupported fields instead of returning NULL as Spark does. + .exclude("unsupported fmt fields for trunc/date_trunc results null") + // Native date_trunc may produce incorrect results for historical timestamps with + // non-UTC timezones due to timezone handling differences in the DataFusion engine. + .exclude("SPARK-30766: date_trunc of old timestamps to hours and days") + + enableSuite[AuronMathFunctionsSuite] + // Native acosh uses a different floating-point formula than Spark's StrictMath.log, + // producing results that differ at the last ULP for certain edge-case inputs. + .exclude("acosh") + + enableSuite[AuronMiscFunctionsSuite] + + enableSuite[AuronStringFunctionsSuite] + // Spark 4 adds the threshold argument, but native levenshtein currently supports only + // two arguments. + .exclude("string Levenshtein distance") + // Native substr does not support BinaryType inputs. + .exclude("string / binary substring function") + override def getSQLQueryTestSettings: SQLQueryTestSettings = new SQLQueryTestSettings { override def getResourceFilePath: String = "" override def getSupportedSQLQueryTests: Set[String] = Set.empty diff --git a/auron-spark-tests/spark40/src/test/scala/org/apache/spark/sql/AuronDataFrameFunctionsSuite.scala b/auron-spark-tests/spark40/src/test/scala/org/apache/spark/sql/AuronDataFrameFunctionsSuite.scala new file mode 100644 index 000000000..98448e333 --- /dev/null +++ b/auron-spark-tests/spark40/src/test/scala/org/apache/spark/sql/AuronDataFrameFunctionsSuite.scala @@ -0,0 +1,19 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql + +class AuronDataFrameFunctionsSuite extends DataFrameFunctionsSuite with SparkQueryTestsBase {} diff --git a/auron-spark-tests/spark40/src/test/scala/org/apache/spark/sql/AuronDateFunctionsSuite.scala b/auron-spark-tests/spark40/src/test/scala/org/apache/spark/sql/AuronDateFunctionsSuite.scala new file mode 100644 index 000000000..dddef76fb --- /dev/null +++ b/auron-spark-tests/spark40/src/test/scala/org/apache/spark/sql/AuronDateFunctionsSuite.scala @@ -0,0 +1,19 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql + +class AuronDateFunctionsSuite extends DateFunctionsSuite with SparkQueryTestsBase {} diff --git a/auron-spark-tests/spark40/src/test/scala/org/apache/spark/sql/AuronMathFunctionsSuite.scala b/auron-spark-tests/spark40/src/test/scala/org/apache/spark/sql/AuronMathFunctionsSuite.scala new file mode 100644 index 000000000..eb2d96a9b --- /dev/null +++ b/auron-spark-tests/spark40/src/test/scala/org/apache/spark/sql/AuronMathFunctionsSuite.scala @@ -0,0 +1,19 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql + +class AuronMathFunctionsSuite extends MathFunctionsSuite with SparkQueryTestsBase {} diff --git a/auron-spark-tests/spark40/src/test/scala/org/apache/spark/sql/AuronMiscFunctionsSuite.scala b/auron-spark-tests/spark40/src/test/scala/org/apache/spark/sql/AuronMiscFunctionsSuite.scala new file mode 100644 index 000000000..a50ff3367 --- /dev/null +++ b/auron-spark-tests/spark40/src/test/scala/org/apache/spark/sql/AuronMiscFunctionsSuite.scala @@ -0,0 +1,19 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql + +class AuronMiscFunctionsSuite extends MiscFunctionsSuite with SparkQueryTestsBase {} diff --git a/auron-spark-tests/spark40/src/test/scala/org/apache/spark/sql/AuronStringFunctionsSuite.scala b/auron-spark-tests/spark40/src/test/scala/org/apache/spark/sql/AuronStringFunctionsSuite.scala new file mode 100644 index 000000000..bd621ba17 --- /dev/null +++ b/auron-spark-tests/spark40/src/test/scala/org/apache/spark/sql/AuronStringFunctionsSuite.scala @@ -0,0 +1,19 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql + +class AuronStringFunctionsSuite extends StringFunctionsSuite with SparkQueryTestsBase {} diff --git a/auron-spark-tests/spark41/src/test/scala/org/apache/auron/utils/AuronSparkTestSettings.scala b/auron-spark-tests/spark41/src/test/scala/org/apache/auron/utils/AuronSparkTestSettings.scala index 052cca5d1..a04eaebe1 100644 --- a/auron-spark-tests/spark41/src/test/scala/org/apache/auron/utils/AuronSparkTestSettings.scala +++ b/auron-spark-tests/spark41/src/test/scala/org/apache/auron/utils/AuronSparkTestSettings.scala @@ -16,7 +16,55 @@ */ package org.apache.auron.utils +import org.apache.spark.sql._ + class AuronSparkTestSettings extends SparkTestSettings { + { + // Use Arrow's unsafe implementation. + System.setProperty("arrow.allocation.manager.type", "Unsafe") + } + + enableSuite[AuronDataFrameFunctionsSuite] + // Native execution wraps SparkRuntimeException from validation failures in SparkException. + .exclude("map with arrays") + .exclude("map_concat function") + .exclude("SPARK-24734: Fix containsNull of Concat for array type") + .exclude("array_insert functions") + .exclude("transform keys function - Invalid lambda functions and exceptions") + // Native reverse only supports string inputs; array inputs fail with unsupported List type. + .exclude("reverse function - array for primitive type not containing null") + .exclude("reverse function - array for primitive type containing null") + .exclude("reverse function - array for non-primitive type") + // Native flatten can panic when child arrays have different containsNull metadata. + .exclude("flatten function") + + enableSuite[AuronDateFunctionsSuite] + // Native execution wraps Spark parsing/format validation exceptions in SparkException. + .exclude("function to_date") + .exclude("unix_timestamp") + .exclude("to_unix_timestamp") + // Native date_trunc does not support all Spark aliases such as "yy". + .exclude("function date_trunc") + // Native date_trunc throws for unsupported fields instead of returning NULL as Spark does. + .exclude("unsupported fmt fields for trunc/date_trunc results null") + // Native date_trunc may produce incorrect results for historical timestamps with + // non-UTC timezones due to timezone handling differences in the DataFusion engine. + .exclude("SPARK-30766: date_trunc of old timestamps to hours and days") + + enableSuite[AuronMathFunctionsSuite] + // Native acosh uses a different floating-point formula than Spark's StrictMath.log, + // producing results that differ at the last ULP for certain edge-case inputs. + .exclude("acosh") + + enableSuite[AuronMiscFunctionsSuite] + + enableSuite[AuronStringFunctionsSuite] + // Spark 4 adds the threshold argument, but native levenshtein currently supports only + // two arguments. + .exclude("string Levenshtein distance") + // Native substr does not support BinaryType inputs. + .exclude("string / binary substring function") + override def getSQLQueryTestSettings: SQLQueryTestSettings = new SQLQueryTestSettings { override def getResourceFilePath: String = "" override def getSupportedSQLQueryTests: Set[String] = Set.empty diff --git a/auron-spark-tests/spark41/src/test/scala/org/apache/spark/sql/AuronDataFrameFunctionsSuite.scala b/auron-spark-tests/spark41/src/test/scala/org/apache/spark/sql/AuronDataFrameFunctionsSuite.scala new file mode 100644 index 000000000..98448e333 --- /dev/null +++ b/auron-spark-tests/spark41/src/test/scala/org/apache/spark/sql/AuronDataFrameFunctionsSuite.scala @@ -0,0 +1,19 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql + +class AuronDataFrameFunctionsSuite extends DataFrameFunctionsSuite with SparkQueryTestsBase {} diff --git a/auron-spark-tests/spark41/src/test/scala/org/apache/spark/sql/AuronDateFunctionsSuite.scala b/auron-spark-tests/spark41/src/test/scala/org/apache/spark/sql/AuronDateFunctionsSuite.scala new file mode 100644 index 000000000..dddef76fb --- /dev/null +++ b/auron-spark-tests/spark41/src/test/scala/org/apache/spark/sql/AuronDateFunctionsSuite.scala @@ -0,0 +1,19 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql + +class AuronDateFunctionsSuite extends DateFunctionsSuite with SparkQueryTestsBase {} diff --git a/auron-spark-tests/spark41/src/test/scala/org/apache/spark/sql/AuronMathFunctionsSuite.scala b/auron-spark-tests/spark41/src/test/scala/org/apache/spark/sql/AuronMathFunctionsSuite.scala new file mode 100644 index 000000000..eb2d96a9b --- /dev/null +++ b/auron-spark-tests/spark41/src/test/scala/org/apache/spark/sql/AuronMathFunctionsSuite.scala @@ -0,0 +1,19 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql + +class AuronMathFunctionsSuite extends MathFunctionsSuite with SparkQueryTestsBase {} diff --git a/auron-spark-tests/spark41/src/test/scala/org/apache/spark/sql/AuronMiscFunctionsSuite.scala b/auron-spark-tests/spark41/src/test/scala/org/apache/spark/sql/AuronMiscFunctionsSuite.scala new file mode 100644 index 000000000..a50ff3367 --- /dev/null +++ b/auron-spark-tests/spark41/src/test/scala/org/apache/spark/sql/AuronMiscFunctionsSuite.scala @@ -0,0 +1,19 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql + +class AuronMiscFunctionsSuite extends MiscFunctionsSuite with SparkQueryTestsBase {} diff --git a/auron-spark-tests/spark41/src/test/scala/org/apache/spark/sql/AuronStringFunctionsSuite.scala b/auron-spark-tests/spark41/src/test/scala/org/apache/spark/sql/AuronStringFunctionsSuite.scala new file mode 100644 index 000000000..bd621ba17 --- /dev/null +++ b/auron-spark-tests/spark41/src/test/scala/org/apache/spark/sql/AuronStringFunctionsSuite.scala @@ -0,0 +1,19 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql + +class AuronStringFunctionsSuite extends StringFunctionsSuite with SparkQueryTestsBase {}