Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
22 commits
Select commit Hold shift + click to select a range
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions doc/whats-new.rst
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,10 @@ v2026.05.0 (unreleased)

New Features
~~~~~~~~~~~~
- Added `PyArrowCapsule interface <https://arrow.apache.org/docs/format/CDataInterface/PyCapsuleInterface.html#arrow-pycapsule-interface>`_
to :py:class:`DataArray` (``__arrow_c_schema__`` and ``__arrow_c_stream__``), enabling near zero-copy
export to pyarrow, polars or duckdb.
By `Jules Chéron <https://github.com/jules-ch>`_.
- Following pandas, xarray's
:py:class:`~xarray.core.accessor_dt.DatetimeAccessor` now supports
:py:attr:`~xarray.core.accessor_dt.DatetimeAccessor.day_of_week` and
Expand Down
2 changes: 2 additions & 0 deletions pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,7 @@ io = [
"cftime",
"pooch",
]
arrow = ["pyarrow"]
etc = ["sparse>=0.15"]
parallel = ["dask[complete]"]
viz = ["cartopy>=0.24", "matplotlib>=3.10", "nc-time-axis", "seaborn"]
Expand Down Expand Up @@ -157,6 +158,7 @@ module = [
"opt_einsum.*",
"pint.*",
"pooch.*",
"polars.*",
"pyarrow.*",
"pydap.*",
"seaborn.*",
Expand Down
113 changes: 113 additions & 0 deletions xarray/core/dataarray.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@

import copy
import datetime
import json
import warnings
from collections.abc import (
Callable,
Expand Down Expand Up @@ -73,6 +74,7 @@
as_compatible_data,
as_variable,
)
from xarray.namedarray.pycompat import is_chunked_array
from xarray.plot.accessor import DataArrayPlotAccessor
from xarray.plot.utils import _get_units_from_attrs
from xarray.structure import alignment
Expand Down Expand Up @@ -255,6 +257,21 @@ def __setitem__(self, key, value) -> None:
_THIS_ARRAY = ReprObject("<this-array>")


class _NumpyEncoder(json.JSONEncoder):
"""Encode numpy value in Arrow schema metadata"""

def default(self, obj: Any) -> Any:
if isinstance(obj, np.integer):
return int(obj)
if isinstance(obj, np.floating):
return float(obj)
if isinstance(obj, np.bool_):
return bool(obj)
if isinstance(obj, np.ndarray):
return obj.tolist()
return super().default(obj)


class DataArray(
AbstractArray,
DataWithCoords,
Expand Down Expand Up @@ -477,6 +494,102 @@ def __init__(

self._close = None

def __arrow_c_schema__(self):
try:
import pyarrow as pa
except ImportError:
raise ImportError(
"pyarrow is required to export via the Arrow PyCapsule Interface."
) from None

values_column = self.name or "values"

fields = []
for name, coord in self._coords.items():
arrow_dtype = pa.from_numpy_dtype(coord.dtype)
fields.append(pa.field(str(name), arrow_dtype))
Comment thread
dcherian marked this conversation as resolved.

fields.append(
pa.field(str(values_column), pa.from_numpy_dtype(self._variable.dtype))
)

xarray_metadata = {
"name": self.name,
"dims": list(self._variable.dims),
"shape": list(self._variable.shape),
"attrs": self.attrs,
"coords": {
str(name): variable.to_dict(data=False)
for name, variable in self._coords.items()
},
Comment thread
dcherian marked this conversation as resolved.
}
schema_metadata = {
b"xarray:arrow_schema_version": b"v1",
b"xarray": json.dumps(xarray_metadata, cls=_NumpyEncoder).encode(),
}

schema = pa.schema(fields, metadata=schema_metadata)
return schema.__arrow_c_schema__()

def __arrow_c_stream__(self, requested_schema: Any = None) -> Any:
"""Export the DataArray through the Arrow PyCapsule Interface.

https://arrow.apache.org/docs/dev/format/CDataInterface/PyCapsuleInterface.html
"""
try:
import pyarrow as pa
except ImportError:
raise ImportError(
"pyarrow is required to export via the Arrow PyCapsule Interface."
) from None

if is_chunked_array(self._variable._data):
raise ValueError(
"Chunked DataArray must be computed first, use: DataArray.compute()"
)

values = self._variable.values
dims = self._variable.dims
shape = self._variable.shape

values_column = self.name or "values"

if not values.flags.c_contiguous:
values = np.ascontiguousarray(values)
Comment on lines +557 to +558

Copy link
Copy Markdown
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think we can only use values.ravel down there to ensure contiguous array.


columns: dict[Hashable, pa.Array] = {}
for name, coord in self._coords.items():
# Broadcast each coordinate up to the full data shape so that 1D
# dimension coordinates and N-D (e.g. curvilinear) coordinates
# flatten consistently with the data values.

# Order axes based on Variable dims
dim_order = tuple(
coord.dims.index(dim) for dim in dims if dim in coord.dims
)

# Reorder coords values to variable dim order
ordered_coords = coord.values.transpose(dim_order)

# Expand coord dims
# coord dims (x, y) variable dims (x,y,z) -> (x, y, 1)
# NOTE: Insert a length-1 axis for each data dim missing for coordinates
# (slice(None) keeps an existing axis, np.newaxis adds one)
indexer = tuple(
slice(None) if dim in coord.dims else np.newaxis for dim in dims
)
expanded_coords = ordered_coords[indexer]

# Broadcast to full flattened shape (x, y, 1) -> (x, y, z)
columns[name] = pa.array(np.broadcast_to(expanded_coords, shape).ravel())

columns[values_column] = pa.array(np.ravel(values))

schema = pa.schema(self)

table = pa.table(columns, schema=schema)
return table.__arrow_c_stream__(requested_schema)

@classmethod
def _construct_direct(
cls,
Expand Down
1 change: 1 addition & 0 deletions xarray/tests/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -174,6 +174,7 @@ def get_dask_chunkmanager():
has_iris, requires_iris = _importorskip("iris")
has_numbagg, requires_numbagg = _importorskip("numbagg")
has_pyarrow, requires_pyarrow = _importorskip("pyarrow")
has_polars, requires_polars = _importorskip("polars")
with warnings.catch_warnings():
warnings.filterwarnings(
"ignore",
Expand Down
Loading
Loading