From 1ddf7b3693c758450b861d5fa1ab4376ba20f52f Mon Sep 17 00:00:00 2001 From: Friedrich Knuth Date: Mon, 16 Sep 2019 11:21:48 -0700 Subject: [PATCH 1/7] convert DataArray to DataSet before combine_by_coords --- xarray/core/combine.py | 25 +++++++++++++++++++++---- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/xarray/core/combine.py b/xarray/core/combine.py index e35bb51e030..0ab4d444bbf 100644 --- a/xarray/core/combine.py +++ b/xarray/core/combine.py @@ -470,7 +470,7 @@ def vars_as_keys(ds): def combine_by_coords( - datasets, + objects, compat="no_conflicts", data_vars="all", coords="different", @@ -483,7 +483,9 @@ def combine_by_coords( This method attempts to combine a group of datasets along any number of dimensions into a single entity by inspecting coords and metadata and using - a combination of concat and merge. + a combination of concat and merge. If passed a DataArray, the object + will be converted to Dataset before being combined with the remaining datasets + in the iterable. Will attempt to order the datasets such that the values in their dimension coordinates are monotonic along all dimensions. If it cannot determine the @@ -503,8 +505,7 @@ def combine_by_coords( Parameters ---------- - datasets : sequence of xarray.Dataset - Dataset objects to combine. + objects : sequence of xarray.Dataset or xarray.DataArray objects compat : {'identical', 'equals', 'broadcast_equals', 'no_conflicts', 'override'}, optional String indicating how to compare variables of the same name for @@ -581,6 +582,22 @@ def combine_by_coords( temperature (x) float64 11.04 23.57 20.77 ... """ + # Convert dict like objects to Dataset(s) + from .dataarray import DataArray + from .dataset import Dataset + + dict_like_objects = list() + for obj in objects: + if not (isinstance(obj, (DataArray, Dataset))): + raise TypeError( + "objects must be an iterable containing only " + "Dataset(s) and/or DataArray(s)" + ) + + obj = obj.to_dataset() if isinstance(obj, DataArray) else obj + dict_like_objects.append(obj) + datasets = dict_like_objects + # Group by data vars sorted_datasets = sorted(datasets, key=vars_as_keys) grouped_by_vars = itertools.groupby(sorted_datasets, key=vars_as_keys) From cf03a7366cf5c0b78d8865ea9d92c9900c4d9784 Mon Sep 17 00:00:00 2001 From: Friedrich Knuth Date: Mon, 16 Sep 2019 11:42:01 -0700 Subject: [PATCH 2/7] fix formatting --- xarray/core/combine.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/xarray/core/combine.py b/xarray/core/combine.py index 0ab4d444bbf..712904d2d99 100644 --- a/xarray/core/combine.py +++ b/xarray/core/combine.py @@ -483,7 +483,7 @@ def combine_by_coords( This method attempts to combine a group of datasets along any number of dimensions into a single entity by inspecting coords and metadata and using - a combination of concat and merge. If passed a DataArray, the object + a combination of concat and merge. If passed a DataArray, the object will be converted to Dataset before being combined with the remaining datasets in the iterable. @@ -585,7 +585,7 @@ def combine_by_coords( # Convert dict like objects to Dataset(s) from .dataarray import DataArray from .dataset import Dataset - + dict_like_objects = list() for obj in objects: if not (isinstance(obj, (DataArray, Dataset))): @@ -597,7 +597,7 @@ def combine_by_coords( obj = obj.to_dataset() if isinstance(obj, DataArray) else obj dict_like_objects.append(obj) datasets = dict_like_objects - + # Group by data vars sorted_datasets = sorted(datasets, key=vars_as_keys) grouped_by_vars = itertools.groupby(sorted_datasets, key=vars_as_keys) From 8a84ec05239b086f20d6dc1c2a3c7c203fe5b359 Mon Sep 17 00:00:00 2001 From: Friedrich Knuth Date: Mon, 16 Sep 2019 13:32:46 -0700 Subject: [PATCH 3/7] add helper function --- xarray/core/combine.py | 32 +++++++++++++++++--------------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/xarray/core/combine.py b/xarray/core/combine.py index 712904d2d99..e029f13d248 100644 --- a/xarray/core/combine.py +++ b/xarray/core/combine.py @@ -468,6 +468,22 @@ def combine_nested( def vars_as_keys(ds): return tuple(sorted(ds)) +def _iter_objects_to_ds(objects): + # Convert sequence of Dataset and/or DataArray objects to Dataset(s) + from .dataarray import DataArray + from .dataset import Dataset + + datasets = list() + for obj in objects: + if not (isinstance(obj, (DataArray, Dataset))): + raise TypeError( + "objects must be an iterable containing only " + "Dataset(s) and/or DataArray(s)" + ) + + obj = obj.to_dataset() if isinstance(obj, DataArray) else obj + datasets.append(obj) + return datasets def combine_by_coords( objects, @@ -582,21 +598,7 @@ def combine_by_coords( temperature (x) float64 11.04 23.57 20.77 ... """ - # Convert dict like objects to Dataset(s) - from .dataarray import DataArray - from .dataset import Dataset - - dict_like_objects = list() - for obj in objects: - if not (isinstance(obj, (DataArray, Dataset))): - raise TypeError( - "objects must be an iterable containing only " - "Dataset(s) and/or DataArray(s)" - ) - - obj = obj.to_dataset() if isinstance(obj, DataArray) else obj - dict_like_objects.append(obj) - datasets = dict_like_objects + datasets = _iter_objects_to_ds(objects) # Group by data vars sorted_datasets = sorted(datasets, key=vars_as_keys) From ac280e39030a741a36fca2ad8d085c0235a298e8 Mon Sep 17 00:00:00 2001 From: Friedrich Knuth Date: Tue, 17 Sep 2019 12:51:17 -0700 Subject: [PATCH 4/7] convert DataArray to DataSet before auto_combine --- xarray/core/combine.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/xarray/core/combine.py b/xarray/core/combine.py index e029f13d248..5846ef4f817 100644 --- a/xarray/core/combine.py +++ b/xarray/core/combine.py @@ -651,7 +651,7 @@ def combine_by_coords( def auto_combine( - datasets, + objects, concat_dim="_not_supplied", compat="no_conflicts", data_vars="all", @@ -666,6 +666,9 @@ def auto_combine( This entire function is deprecated in favour of ``combine_nested`` and ``combine_by_coords``. + If passed a DataArray, the object will be converted to Dataset before being + combined with the remaining datasets in the iterable. + This method attempts to combine a list of datasets into a single entity by inspecting metadata and using a combination of concat and merge. It does not concatenate along more than one dimension or sort data under @@ -678,8 +681,7 @@ def auto_combine( Parameters ---------- - datasets : sequence of xarray.Dataset - Dataset objects to merge. + objects : sequence of xarray.Dataset or xarray.DataArray objects concat_dim : str or DataArray or Index, optional Dimension along which to concatenate variables, as used by :py:func:`xarray.concat`. You only need to provide this argument if @@ -732,6 +734,8 @@ def auto_combine( Dataset.merge """ + datasets = _iter_objects_to_ds(objects) + if not from_openmfds: basic_msg = dedent( """\ From ffbc7b81181cf9b747cbbea77a32968e3b4cd548 Mon Sep 17 00:00:00 2001 From: Friedrich Knuth Date: Tue, 17 Sep 2019 13:22:48 -0700 Subject: [PATCH 5/7] convert DataArray to DataSet before combine_nested --- xarray/core/combine.py | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/xarray/core/combine.py b/xarray/core/combine.py index 5846ef4f817..d175f98afed 100644 --- a/xarray/core/combine.py +++ b/xarray/core/combine.py @@ -310,7 +310,7 @@ def _nested_combine( def combine_nested( - datasets, + objects, concat_dim, compat="no_conflicts", data_vars="all", @@ -322,6 +322,9 @@ def combine_nested( Explicitly combine an N-dimensional grid of datasets into one by using a succession of concat and merge operations along each dimension of the grid. + If passed a DataArray, the object will be converted to Dataset before being + combined with the remaining datasets in the iterable. + Does not sort the supplied datasets under any circumstances, so the datasets must be passed in the order you wish them to be concatenated. It does align coordinates, but different variables on datasets can cause it to @@ -449,6 +452,9 @@ def combine_nested( merge auto_combine """ + + datasets = _iter_objects_to_ds(objects) + if isinstance(concat_dim, (str, DataArray)) or concat_dim is None: concat_dim = [concat_dim] @@ -475,14 +481,21 @@ def _iter_objects_to_ds(objects): datasets = list() for obj in objects: - if not (isinstance(obj, (DataArray, Dataset))): + # Handle nested lists of DataArrays or Datasets for combine_nested + if isinstance(obj, (list, tuple)): + nested_datasets = _iter_objects_to_ds(obj) + datasets.append(nested_datasets) + + elif (isinstance(obj, (DataArray, Dataset))): + obj = obj.to_dataset() if isinstance(obj, DataArray) else obj + datasets.append(obj) + + else: raise TypeError( "objects must be an iterable containing only " "Dataset(s) and/or DataArray(s)" ) - obj = obj.to_dataset() if isinstance(obj, DataArray) else obj - datasets.append(obj) return datasets def combine_by_coords( From 7200c94556713c1495b81bdfa0cc44c94dd12627 Mon Sep 17 00:00:00 2001 From: Friedrich Knuth Date: Tue, 17 Sep 2019 13:39:57 -0700 Subject: [PATCH 6/7] Update whats-new.rst --- doc/whats-new.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/doc/whats-new.rst b/doc/whats-new.rst index 492c9279e6b..4490dcf63e2 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -103,6 +103,11 @@ New functions/methods Enhancements ~~~~~~~~~~~~ +- Functions :py:func:`~xarray.combine.combine_nested`, :py:func:`~xarray.combine.combine_by_coords`, + and :py:func:`~xarray.combine.auto_combine` now accept :py:class:`~xarray.DataArray` as input. + The DataArray will be converted to :py:class:`~xarray.DataSet` before proceeding. + By `Friedrich Knuth `_. + - Multiple enhancements to :py:func:`~xarray.concat` and :py:func:`~xarray.open_mfdataset`. - Added ``compat='override'``. When merging, this option picks the variable from the first dataset From 53afe58ae12f1b0e5873638b7a05db0845862b8c Mon Sep 17 00:00:00 2001 From: Friedrich Knuth Date: Tue, 17 Sep 2019 13:53:52 -0700 Subject: [PATCH 7/7] Fix formatting --- xarray/core/combine.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/xarray/core/combine.py b/xarray/core/combine.py index d175f98afed..da648936c16 100644 --- a/xarray/core/combine.py +++ b/xarray/core/combine.py @@ -322,7 +322,7 @@ def combine_nested( Explicitly combine an N-dimensional grid of datasets into one by using a succession of concat and merge operations along each dimension of the grid. - If passed a DataArray, the object will be converted to Dataset before being + If passed a DataArray, the object will be converted to Dataset before being combined with the remaining datasets in the iterable. Does not sort the supplied datasets under any circumstances, so the @@ -474,6 +474,7 @@ def combine_nested( def vars_as_keys(ds): return tuple(sorted(ds)) + def _iter_objects_to_ds(objects): # Convert sequence of Dataset and/or DataArray objects to Dataset(s) from .dataarray import DataArray @@ -486,7 +487,7 @@ def _iter_objects_to_ds(objects): nested_datasets = _iter_objects_to_ds(obj) datasets.append(nested_datasets) - elif (isinstance(obj, (DataArray, Dataset))): + elif isinstance(obj, (DataArray, Dataset)): obj = obj.to_dataset() if isinstance(obj, DataArray) else obj datasets.append(obj) @@ -498,6 +499,7 @@ def _iter_objects_to_ds(objects): return datasets + def combine_by_coords( objects, compat="no_conflicts", @@ -679,7 +681,7 @@ def auto_combine( This entire function is deprecated in favour of ``combine_nested`` and ``combine_by_coords``. - If passed a DataArray, the object will be converted to Dataset before being + If passed a DataArray, the object will be converted to Dataset before being combined with the remaining datasets in the iterable. This method attempts to combine a list of datasets into a single entity by