Merge remote-tracking branch 'apache/main' into alamb/field_metadata2

Author: alamb

datafusion/functions-nested/src/position.rs

@@ -52,7 +52,7 @@ make_udf_expr_and_func!(
 
 #[user_doc(
     doc_section(label = "Array Functions"),
-    description = "Returns the position of the first occurrence of the specified element in the array.",
+    description = "Returns the position of the first occurrence of the specified element in the array, or NULL if not found.",
     syntax_example = "array_position(array, element)\narray_position(array, element, index)",
     sql_example = r#"```sql
 > select array_position([1, 2, 2, 3, 1, 4], 2);
@@ -76,7 +76,10 @@ make_udf_expr_and_func!(
         name = "element",
         description = "Element to search for position in the array."
     ),
-    argument(name = "index", description = "Index at which to start searching.")
+    argument(
+        name = "index",
+        description = "Index at which to start searching (1-indexed)."
+    )
 )]
 #[derive(Debug)]
 pub struct ArrayPosition {
@@ -170,7 +173,7 @@ fn general_position_dispatch<O: OffsetSizeTrait>(args: &[ArrayRef]) -> Result<Ar
     // if `start_from` index is out of bounds, return error
     for (arr, &from) in list_array.iter().zip(arr_from.iter()) {
         if let Some(arr) = arr {
-            if from < 0 || from as usize >= arr.len() {
+            if from < 0 || from as usize > arr.len() {
                 return internal_err!("start_from index out of bounds");
             }
         } else {

datafusion/sqllogictest/test_files/array.slt

@@ -3367,10 +3367,16 @@ select array_concat(make_array(column3), column1, column2) from arrays_values_v2
 ## array_position (aliases: `list_position`, `array_indexof`, `list_indexof`)
 
 ## array_position with NULL (follow PostgreSQL)
-#query I
-#select array_position([1, 2, 3, 4, 5], null), array_position(NULL, 1);
-#----
-#NULL NULL
+query II
+select array_position([1, 2, 3, 4, 5], arrow_cast(NULL, 'Int64')), array_position(arrow_cast(NULL, 'List(Int64)'), 1);
+----
+NULL NULL
+
+# array_position with no match (incl. empty array) returns NULL
+query II
+select array_position([], 1), array_position([2], 1);
+----
+NULL NULL
 
 # array_position scalar function #1
 query III

docs/source/library-user-guide/upgrading.md

@@ -50,6 +50,60 @@ See [#16317] for details.
 
 ## DataFusion `48.0.0`
 
+### `Expr::WindowFunction` is now `Box`ed
+
+`Expr::WindowFunction` is now a `Box<WindowFunction>` instead of a `WindowFunction` directly.
+This change was made to reduce the size of `Expr` and improve performance when
+planning queries (see [details on #16207]).
+
+This is a breaking change, so you will need to update your code if you match
+on `Expr::WindowFunction` directly. For example, if you have code like this:
+
+```rust
+# /* comment to avoid running
+match expr {
+  Expr::WindowFunction(WindowFunction {
+    params:
+      WindowFunctionParams {
+       partition_by,
+       order_by,
+      ..
+    }
+  }) => {
+    // Use partition_by and order_by as needed
+  }
+  _ => {
+    // other expr
+  }
+}
+# */
+```
+
+You will need to change it to:
+
+```rust
+# /* comment to avoid running
+match expr {
+  Expr::WindowFunction(window_fun) => {
+    let WindowFunction {
+      fun,
+      params: WindowFunctionParams {
+        args,
+        partition_by,
+        ..
+        },
+    } = window_fun.as_ref();
+    // Use partition_by and order_by as needed
+  }
+  _ => {
+    // other expr
+  }
+}
+#  */
+```
+
+[details on #16207]: https://github.com/apache/datafusion/pull/16207#issuecomment-2922659103
+
 ### The `VARCHAR` SQL type is now represented as `Utf8View` in Arrow.
 
 The mapping of the SQL `VARCHAR` type has been changed from `Utf8` to `Utf8View`

docs/source/user-guide/sql/scalar_functions.md

@@ -3142,7 +3142,7 @@ array_pop_front(array)
 
 ### `array_position`
 
-Returns the position of the first occurrence of the specified element in the array.
+Returns the position of the first occurrence of the specified element in the array, or NULL if not found.
 
 ```sql
 array_position(array, element)
@@ -3153,7 +3153,7 @@ array_position(array, element, index)
 
 - **array**: Array expression. Can be a constant, column, or function, and any combination of array operators.
 - **element**: Element to search for position in the array.
-- **index**: Index at which to start searching.
+- **index**: Index at which to start searching (1-indexed).
 
 #### Example