-
-
Notifications
You must be signed in to change notification settings - Fork 1.3k
feat: add __arrow_c_stream__ function #11338
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Open
jules-ch
wants to merge
22
commits into
pydata:main
Choose a base branch
from
jules-ch:arrow-pycapsule-datarray
base: main
Could not load branches
Branch not found: {{ refName }}
Loading
Could not load tags
Nothing to show
Loading
Are you sure you want to change the base?
Some commits from the old base branch may be removed from the timeline,
and old review comments may become outdated.
Open
Changes from all commits
Commits
Show all changes
22 commits
Select commit
Hold shift + click to select a range
f70df33
feat: add __arrow_c_stream__ function
jules-ch 11fd74b
chore: typing
jules-ch 8b0c828
chore: rename class Test
jules-ch ac02132
chore: typo
jules-ch f17c813
chore: remove call to action
jules-ch 74a7251
chore: fix mypy
jules-ch e69b4f4
fix: use values to ensure numpy array
jules-ch 57b836d
chore: add test for dask
jules-ch e7cc3f1
add polars to ignore missing imports
jules-ch ba824b4
chore: add pyarrow schema method + json encoding of attrs
jules-ch ecadc76
chore: use directly _coords without going through self.coords + dim m…
jules-ch 52fed6b
chore: add tests for pyarrow __arrow_c_schema__
jules-ch 5872890
chore: add what's new entry
jules-ch 5c9c8c0
feat: update to support curvilinear coordinates
jules-ch 276c394
chore: add docstring
jules-ch 64b695d
chore: add tests for curvilinear coords and out of order dims
jules-ch d998785
feat: add shape to xarray schema metadata
jules-ch 1eda714
Update xarray/core/dataarray.py
jules-ch ad8c76d
Update arrow schema version key
jules-ch 1054c49
fix: raise ValueError on chunked array
jules-ch 3b205dc
Merge branch 'main' into arrow-pycapsule-datarray
dcherian 4b08909
Merge branch 'main' into arrow-pycapsule-datarray
dcherian File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -2,6 +2,7 @@ | |
|
|
||
| import copy | ||
| import datetime | ||
| import json | ||
| import warnings | ||
| from collections.abc import ( | ||
| Callable, | ||
|
|
@@ -73,6 +74,7 @@ | |
| as_compatible_data, | ||
| as_variable, | ||
| ) | ||
| from xarray.namedarray.pycompat import is_chunked_array | ||
| from xarray.plot.accessor import DataArrayPlotAccessor | ||
| from xarray.plot.utils import _get_units_from_attrs | ||
| from xarray.structure import alignment | ||
|
|
@@ -255,6 +257,21 @@ def __setitem__(self, key, value) -> None: | |
| _THIS_ARRAY = ReprObject("<this-array>") | ||
|
|
||
|
|
||
| class _NumpyEncoder(json.JSONEncoder): | ||
| """Encode numpy value in Arrow schema metadata""" | ||
|
|
||
| def default(self, obj: Any) -> Any: | ||
| if isinstance(obj, np.integer): | ||
| return int(obj) | ||
| if isinstance(obj, np.floating): | ||
| return float(obj) | ||
| if isinstance(obj, np.bool_): | ||
| return bool(obj) | ||
| if isinstance(obj, np.ndarray): | ||
| return obj.tolist() | ||
| return super().default(obj) | ||
|
|
||
|
|
||
| class DataArray( | ||
| AbstractArray, | ||
| DataWithCoords, | ||
|
|
@@ -477,6 +494,102 @@ def __init__( | |
|
|
||
| self._close = None | ||
|
|
||
| def __arrow_c_schema__(self): | ||
| try: | ||
| import pyarrow as pa | ||
| except ImportError: | ||
| raise ImportError( | ||
| "pyarrow is required to export via the Arrow PyCapsule Interface." | ||
| ) from None | ||
|
|
||
| values_column = self.name or "values" | ||
|
|
||
| fields = [] | ||
| for name, coord in self._coords.items(): | ||
| arrow_dtype = pa.from_numpy_dtype(coord.dtype) | ||
| fields.append(pa.field(str(name), arrow_dtype)) | ||
|
|
||
| fields.append( | ||
| pa.field(str(values_column), pa.from_numpy_dtype(self._variable.dtype)) | ||
| ) | ||
|
|
||
| xarray_metadata = { | ||
| "name": self.name, | ||
| "dims": list(self._variable.dims), | ||
| "shape": list(self._variable.shape), | ||
| "attrs": self.attrs, | ||
| "coords": { | ||
| str(name): variable.to_dict(data=False) | ||
| for name, variable in self._coords.items() | ||
| }, | ||
|
dcherian marked this conversation as resolved.
|
||
| } | ||
| schema_metadata = { | ||
| b"xarray:arrow_schema_version": b"v1", | ||
| b"xarray": json.dumps(xarray_metadata, cls=_NumpyEncoder).encode(), | ||
| } | ||
|
|
||
| schema = pa.schema(fields, metadata=schema_metadata) | ||
| return schema.__arrow_c_schema__() | ||
|
|
||
| def __arrow_c_stream__(self, requested_schema: Any = None) -> Any: | ||
| """Export the DataArray through the Arrow PyCapsule Interface. | ||
|
|
||
| https://arrow.apache.org/docs/dev/format/CDataInterface/PyCapsuleInterface.html | ||
| """ | ||
| try: | ||
| import pyarrow as pa | ||
| except ImportError: | ||
| raise ImportError( | ||
| "pyarrow is required to export via the Arrow PyCapsule Interface." | ||
| ) from None | ||
|
|
||
| if is_chunked_array(self._variable._data): | ||
| raise ValueError( | ||
| "Chunked DataArray must be computed first, use: DataArray.compute()" | ||
| ) | ||
|
|
||
| values = self._variable.values | ||
| dims = self._variable.dims | ||
| shape = self._variable.shape | ||
|
|
||
| values_column = self.name or "values" | ||
|
|
||
| if not values.flags.c_contiguous: | ||
| values = np.ascontiguousarray(values) | ||
|
Comment on lines
+557
to
+558
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I think we can only use |
||
|
|
||
| columns: dict[Hashable, pa.Array] = {} | ||
| for name, coord in self._coords.items(): | ||
| # Broadcast each coordinate up to the full data shape so that 1D | ||
| # dimension coordinates and N-D (e.g. curvilinear) coordinates | ||
| # flatten consistently with the data values. | ||
|
|
||
| # Order axes based on Variable dims | ||
| dim_order = tuple( | ||
| coord.dims.index(dim) for dim in dims if dim in coord.dims | ||
| ) | ||
|
|
||
| # Reorder coords values to variable dim order | ||
| ordered_coords = coord.values.transpose(dim_order) | ||
|
|
||
| # Expand coord dims | ||
| # coord dims (x, y) variable dims (x,y,z) -> (x, y, 1) | ||
| # NOTE: Insert a length-1 axis for each data dim missing for coordinates | ||
| # (slice(None) keeps an existing axis, np.newaxis adds one) | ||
| indexer = tuple( | ||
| slice(None) if dim in coord.dims else np.newaxis for dim in dims | ||
| ) | ||
| expanded_coords = ordered_coords[indexer] | ||
|
|
||
| # Broadcast to full flattened shape (x, y, 1) -> (x, y, z) | ||
| columns[name] = pa.array(np.broadcast_to(expanded_coords, shape).ravel()) | ||
|
|
||
| columns[values_column] = pa.array(np.ravel(values)) | ||
|
|
||
| schema = pa.schema(self) | ||
|
|
||
| table = pa.table(columns, schema=schema) | ||
| return table.__arrow_c_stream__(requested_schema) | ||
|
|
||
| @classmethod | ||
| def _construct_direct( | ||
| cls, | ||
|
|
||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
Uh oh!
There was an error while loading. Please reload this page.