Skip to content

Commit 409efcc

Browse files
VedinVedin
committed
Fixes after sync to 50.0.0
1 parent 78b55f0 commit 409efcc

17 files changed

Lines changed: 149 additions & 77 deletions

File tree

‎Cargo.lock‎

Lines changed: 2 additions & 3 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎Cargo.toml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -173,7 +173,7 @@ recursive = "0.1.1"
173173
regex = "1.11"
174174
rstest = "0.25.0"
175175
serde_json = "1"
176-
sqlparser = { git = "https://github.com/Embucket/datafusion-sqlparser-rs.git", rev = "8d9cbc2669d7fc3d79d7d1d725ac20f1e9bfe10e", features = [
176+
sqlparser = { git = "https://github.com/Embucket/datafusion-sqlparser-rs.git", rev = "75ce913994c7bf8e1e9122f836da882302673bde", features = [
177177
"visitor",
178178
] }
179179
tempfile = "3"

‎datafusion-cli/src/functions.rs‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -322,9 +322,13 @@ pub struct ParquetMetadataFunc {}
322322

323323
impl TableFunctionImpl for ParquetMetadataFunc {
324324
fn call(&self, exprs: &[(Expr, Option<String>)]) -> Result<Arc<dyn TableProvider>> {
325+
if exprs.is_empty() {
326+
return plan_err!("parquet_metadata requires string argument as its input");
327+
}
328+
325329
let filename = match exprs.first() {
326330
Some((Expr::Literal(ScalarValue::Utf8(Some(s)), _), _)) => s, // single quote: parquet_metadata('x.parquet')
327-
Some(Expr::Column(Column { name, .. })) => name, // double quote: parquet_metadata("x.parquet")
331+
Some((Expr::Column(Column { name, .. }), _)) => name, // double quote: parquet_metadata("x.parquet")
328332
_ => {
329333
return plan_err!(
330334
"parquet_metadata requires string argument as its input"
@@ -510,7 +514,7 @@ impl MetadataCacheFunc {
510514
}
511515

512516
impl TableFunctionImpl for MetadataCacheFunc {
513-
fn call(&self, exprs: &[Expr]) -> Result<Arc<dyn TableProvider>> {
517+
fn call(&self, exprs: &[(Expr, Option<String>)]) -> Result<Arc<dyn TableProvider>> {
514518
if !exprs.is_empty() {
515519
return plan_err!("metadata_cache should have no arguments");
516520
}

‎datafusion-examples/examples/simple_udtf.rs‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -133,7 +133,8 @@ struct LocalCsvTableFunc {}
133133

134134
impl TableFunctionImpl for LocalCsvTableFunc {
135135
fn call(&self, exprs: &[(Expr, Option<String>)]) -> Result<Arc<dyn TableProvider>> {
136-
let Some((Expr::Literal(ScalarValue::Utf8(Some(ref path)), _), _)) = exprs.first()
136+
let Some((Expr::Literal(ScalarValue::Utf8(Some(ref path)), _), _)) =
137+
exprs.first()
137138
else {
138139
return plan_err!("read_csv requires at least one string argument");
139140
};

‎datafusion/core/src/execution/session_state.rs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -274,6 +274,8 @@ impl Session for SessionState {
274274
}
275275

276276
impl SessionState {
277+
/// Resolve a [`TableReference`] into a [`ResolvedTableReference`] using
278+
/// the session's configured default catalog and schema.
277279
pub fn resolve_table_ref(
278280
&self,
279281
table_ref: impl Into<TableReference>,

‎datafusion/core/src/physical_planner.rs‎

Lines changed: 4 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -97,7 +97,6 @@ use datafusion_sql::TableReference;
9797
use sqlparser::ast::NullTreatment;
9898

9999
use async_trait::async_trait;
100-
use datafusion_datasource::file_groups::FileGroup;
101100
use datafusion_expr_common::operator::Operator;
102101
use datafusion_physical_plan::async_func::{AsyncFuncExec, AsyncMapper};
103102
use futures::{StreamExt, TryStreamExt};
@@ -1872,8 +1871,6 @@ pub fn create_aggregate_expr_and_maybe_filter(
18721871
)
18731872
}
18741873

1875-
1876-
18771874
/// Transform a PIVOT operation into a more standard Aggregate + Projection plan
18781875
/// For known pivot values, we create a projection that includes "IS NOT DISTINCT FROM" conditions
18791876
///
@@ -1930,7 +1927,7 @@ pub fn transform_pivot_to_aggregate(
19301927
Box::new(Expr::Column(pivot_column.clone())),
19311928
Operator::IsNotDistinctFrom,
19321929
Box::new(Expr::Cast(Cast::new(
1933-
Box::new(Expr::Literal(value.clone())),
1930+
Box::new(Expr::Literal(value.clone(), None)),
19341931
pivot_col_type.clone(),
19351932
))),
19361933
));
@@ -2394,9 +2391,9 @@ impl DefaultPhysicalPlanner {
23942391
if let LogicalPlan::Pivot(pivot) = input.as_ref() {
23952392
if pivot.value_subquery.is_some()
23962393
&& input_exec
2397-
.as_any()
2398-
.downcast_ref::<AggregateExec>()
2399-
.is_some()
2394+
.as_any()
2395+
.downcast_ref::<AggregateExec>()
2396+
.is_some()
24002397
{
24012398
let agg_exec =
24022399
input_exec.as_any().downcast_ref::<AggregateExec>().unwrap();

‎datafusion/expr/src/logical_plan/display.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -759,8 +759,8 @@ impl<'n> TreeNodeVisitor<'n> for PgJsonVisitor<'_, '_> {
759759
mod tests {
760760
use crate::EmptyRelation;
761761
use arrow::datatypes::{DataType, Field, Schema};
762-
use insta::assert_snapshot;
763762
use datafusion_common::{Column, DFSchema, ScalarValue};
763+
use insta::assert_snapshot;
764764
use std::sync::Arc;
765765

766766
use super::*;

‎datafusion/functions-table/src/generate_series.rs‎

Lines changed: 20 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -521,10 +521,13 @@ impl TableFunctionImpl for GenerateSeriesFuncImpl {
521521
}
522522

523523
impl GenerateSeriesFuncImpl {
524-
fn call_int64(&self, exprs: &[Expr]) -> Result<Arc<dyn TableProvider>> {
524+
fn call_int64(
525+
&self,
526+
exprs: &[(Expr, Option<String>)],
527+
) -> Result<Arc<dyn TableProvider>> {
525528
let mut normalize_args = Vec::new();
526-
for (expr_index, expr, _) in exprs.iter().enumerate() {
527-
match expr {
529+
for (expr_index, expr) in exprs.iter().enumerate() {
530+
match &expr.0 {
528531
Expr::Literal(ScalarValue::Null, _) => {}
529532
Expr::Literal(ScalarValue::Int64(Some(n)), _) => normalize_args.push(*n),
530533
other => {
@@ -584,7 +587,10 @@ impl GenerateSeriesFuncImpl {
584587
}))
585588
}
586589

587-
fn call_timestamp(&self, exprs: &[Expr]) -> Result<Arc<dyn TableProvider>> {
590+
fn call_timestamp(
591+
&self,
592+
exprs: &[(Expr, Option<String>)],
593+
) -> Result<Arc<dyn TableProvider>> {
588594
if exprs.len() != 3 {
589595
return plan_err!(
590596
"{} function with timestamps requires exactly 3 arguments",
@@ -593,7 +599,7 @@ impl GenerateSeriesFuncImpl {
593599
}
594600

595601
// Parse start timestamp
596-
let (start_ts, tz) = match &exprs[0] {
602+
let (start_ts, tz) = match &exprs[0].0 {
597603
Expr::Literal(ScalarValue::TimestampNanosecond(ts, tz), _) => {
598604
(*ts, tz.clone())
599605
}
@@ -606,7 +612,7 @@ impl GenerateSeriesFuncImpl {
606612
};
607613

608614
// Parse end timestamp
609-
let end_ts = match &exprs[1] {
615+
let end_ts = match &exprs[1].0 {
610616
Expr::Literal(ScalarValue::Null, _) => None,
611617
Expr::Literal(ScalarValue::TimestampNanosecond(ts, _), _) => *ts,
612618
other => {
@@ -618,7 +624,7 @@ impl GenerateSeriesFuncImpl {
618624
};
619625

620626
// Parse step interval
621-
let step_interval = match &exprs[2] {
627+
let step_interval = match &exprs[2].0 {
622628
Expr::Literal(ScalarValue::Null, _) => None,
623629
Expr::Literal(ScalarValue::IntervalMonthDayNano(interval), _) => *interval,
624630
other => {
@@ -660,7 +666,10 @@ impl GenerateSeriesFuncImpl {
660666
}))
661667
}
662668

663-
fn call_date(&self, exprs: &[Expr]) -> Result<Arc<dyn TableProvider>> {
669+
fn call_date(
670+
&self,
671+
exprs: &[(Expr, Option<String>)],
672+
) -> Result<Arc<dyn TableProvider>> {
664673
if exprs.len() != 3 {
665674
return plan_err!(
666675
"{} function with dates requires exactly 3 arguments",
@@ -675,7 +684,7 @@ impl GenerateSeriesFuncImpl {
675684
)]));
676685

677686
// Parse start date
678-
let start_date = match &exprs[0] {
687+
let start_date = match &exprs[0].0 {
679688
Expr::Literal(ScalarValue::Date32(Some(date)), _) => *date,
680689
Expr::Literal(ScalarValue::Date32(None), _)
681690
| Expr::Literal(ScalarValue::Null, _) => {
@@ -693,7 +702,7 @@ impl GenerateSeriesFuncImpl {
693702
};
694703

695704
// Parse end date
696-
let end_date = match &exprs[1] {
705+
let end_date = match &exprs[1].0 {
697706
Expr::Literal(ScalarValue::Date32(Some(date)), _) => *date,
698707
Expr::Literal(ScalarValue::Date32(None), _)
699708
| Expr::Literal(ScalarValue::Null, _) => {
@@ -711,7 +720,7 @@ impl GenerateSeriesFuncImpl {
711720
};
712721

713722
// Parse step interval
714-
let step_interval = match &exprs[2] {
723+
let step_interval = match &exprs[2].0 {
715724
Expr::Literal(ScalarValue::IntervalMonthDayNano(Some(interval)), _) => {
716725
*interval
717726
}

‎datafusion/functions/src/datetime/to_date.rs‎

Lines changed: 1 addition & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -462,11 +462,7 @@ mod tests {
462462
];
463463
for scalar in test_cases {
464464
let timestamp_to_date_result =
465-
ToDateFunc::new().invoke_with_args(datafusion_expr::ScalarFunctionArgs {
466-
args: vec![ColumnarValue::Scalar(scalar.clone())],
467-
number_rows: 1,
468-
return_type: &DataType::Date32,
469-
});
465+
invoke_to_date_with_args(vec![ColumnarValue::Scalar(scalar.clone())], 1);
470466

471467
match timestamp_to_date_result {
472468
Ok(ColumnarValue::Scalar(ScalarValue::Date32(date_val))) => {

‎datafusion/functions/src/regex/regexpsubstr.rs‎

Lines changed: 65 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,7 @@ use regex::Regex;
3333
use std::any::Any;
3434
use std::sync::{Arc, OnceLock};
3535

36-
#[derive(Debug)]
36+
#[derive(Debug, PartialEq, Eq, Hash)]
3737
pub struct RegexpSubstrFunc {
3838
signature: Signature,
3939
}
@@ -335,7 +335,7 @@ fn compile_regex(regex: &str, flags: Option<&str>) -> Result<Regex, ArrowError>
335335
mod tests {
336336
use crate::regex::regexpsubstr::{regexp_substr, RegexpSubstrFunc};
337337
use arrow::array::{Array, ArrayRef, Int64Array, LargeStringArray, StringArray};
338-
use arrow::datatypes::DataType;
338+
use arrow::datatypes::{DataType, Field};
339339
use datafusion_common::ScalarValue;
340340
use datafusion_expr::{ScalarFunctionArgs, ScalarUDFImpl};
341341
use datafusion_expr_common::columnar_value::ColumnarValue;
@@ -371,14 +371,30 @@ mod tests {
371371
ScalarValue::LargeUtf8 as fn(Option<String>) -> ScalarValue,
372372
),
373373
] {
374+
let args_vec = vec![
375+
ColumnarValue::Scalar(scalar(Some(value.to_string()))),
376+
ColumnarValue::Scalar(scalar(Some(regex.to_string()))),
377+
];
378+
let arg_fields = args_vec
379+
.iter()
380+
.enumerate()
381+
.map(|(idx, arg)| {
382+
Field::new(format!("f_{idx}"), arg.data_type(), true).into()
383+
})
384+
.collect();
374385
let result =
375386
RegexpSubstrFunc::new().invoke_with_args(ScalarFunctionArgs {
376-
args: vec![
377-
ColumnarValue::Scalar(scalar(Some(value.to_string()))),
378-
ColumnarValue::Scalar(scalar(Some(regex.to_string()))),
379-
],
387+
args: args_vec,
388+
arg_fields,
380389
number_rows: 1,
381-
return_type: data_type,
390+
return_field: Arc::new(Field::new(
391+
"f",
392+
data_type.clone(),
393+
true,
394+
)),
395+
config_options: Arc::new(
396+
datafusion_common::config::ConfigOptions::default(),
397+
),
382398
});
383399
match result {
384400
Ok(ColumnarValue::Scalar(
@@ -422,14 +438,26 @@ mod tests {
422438
),
423439
_ => unreachable!(),
424440
};
441+
let args_vec = vec![
442+
ColumnarValue::Array(Arc::new(array_values)),
443+
ColumnarValue::Scalar(regex),
444+
];
445+
let arg_fields = args_vec
446+
.iter()
447+
.enumerate()
448+
.map(|(idx, arg)| {
449+
Field::new(format!("f_{idx}"), arg.data_type(), true).into()
450+
})
451+
.collect();
425452
let result =
426453
RegexpSubstrFunc::new().invoke_with_args(ScalarFunctionArgs {
427-
args: vec![
428-
ColumnarValue::Array(Arc::new(array_values)),
429-
ColumnarValue::Scalar(regex),
430-
],
454+
args: args_vec,
455+
arg_fields,
431456
number_rows: 1,
432-
return_type: data_type,
457+
return_field: Arc::new(Field::new("f", data_type.clone(), true)),
458+
config_options: Arc::new(
459+
datafusion_common::config::ConfigOptions::default(),
460+
),
433461
});
434462
match result {
435463
Ok(ColumnarValue::Array(array)) => {
@@ -511,22 +539,34 @@ mod tests {
511539
ScalarValue::LargeUtf8 as fn(Option<String>) -> ScalarValue,
512540
),
513541
] {
542+
let args_vec = vec![
543+
ColumnarValue::Scalar(scalar(Some(value.to_string()))),
544+
ColumnarValue::Scalar(scalar(Some(regex.to_string()))),
545+
ColumnarValue::Scalar(ScalarValue::Int64(Some(1))),
546+
ColumnarValue::Scalar(ScalarValue::Int64(Some(1))),
547+
ColumnarValue::Scalar(scalar(Some(flags[spos].to_string()))),
548+
ColumnarValue::Scalar(ScalarValue::Int64(Some(group_num[spos]))),
549+
];
550+
let arg_fields = args_vec
551+
.iter()
552+
.enumerate()
553+
.map(|(idx, arg)| {
554+
Field::new(format!("f_{idx}"), arg.data_type(), true).into()
555+
})
556+
.collect();
514557
let result =
515558
RegexpSubstrFunc::new().invoke_with_args(ScalarFunctionArgs {
516-
args: vec![
517-
ColumnarValue::Scalar(scalar(Some(value.to_string()))),
518-
ColumnarValue::Scalar(scalar(Some(regex.to_string()))),
519-
ColumnarValue::Scalar(ScalarValue::Int64(Some(1))),
520-
ColumnarValue::Scalar(ScalarValue::Int64(Some(1))),
521-
ColumnarValue::Scalar(scalar(Some(
522-
flags[spos].to_string(),
523-
))),
524-
ColumnarValue::Scalar(ScalarValue::Int64(Some(
525-
group_num[spos],
526-
))),
527-
],
559+
args: args_vec,
560+
arg_fields,
528561
number_rows: 1,
529-
return_type: data_type,
562+
return_field: Arc::new(Field::new(
563+
"f",
564+
data_type.clone(),
565+
true,
566+
)),
567+
config_options: Arc::new(
568+
datafusion_common::config::ConfigOptions::default(),
569+
),
530570
});
531571
match result {
532572
Ok(ColumnarValue::Scalar(

0 commit comments

Comments
 (0)