Merge branch 'master' into increase-visibility-of-allowed-failures

Author: keewis

.github/ISSUE_TEMPLATE/config.yml

@@ -1,7 +1,10 @@
-blank_issues_enabled: true
+blank_issues_enabled: false
 contact_links:
   - name: Usage question
-    url: https://stackoverflow.com/questions/tagged/python-xarray
-    about: "Post a question on Stack Overflow using the #python-xarray
-    tag. These are regularly reviewed by xarray's maintainers, and questions which
-    include a reproducible example will receive a response."
+    url: https://github.com/pydata/xarray/discussions
+    about: |
+      Ask questions and discuss with other community members here.
+      If you have a question like "How do I concatenate a list of datasets?" then 
+      please include a self-contained reproducible example if possible.
+
+   

.github/workflows/parse_logs.py

@@ -1,21 +1,53 @@
 # type: ignore
+import argparse
+import itertools
 import pathlib
+import textwrap
 
-files = pathlib.Path("logs").rglob("**/*-log")
-files = sorted(filter(lambda x: x.is_file(), files))
+parser = argparse.ArgumentParser()
+parser.add_argument("filepaths", nargs="+", type=pathlib.Path)
+args = parser.parse_args()
+
+filepaths = sorted(p for p in args.filepaths if p.is_file())
+
+
+def extract_short_test_summary_info(lines):
+    up_to_start_of_section = itertools.dropwhile(
+        lambda l: "=== short test summary info ===" not in l,
+        lines,
+    )
+    up_to_section_content = itertools.islice(up_to_start_of_section, 1, None)
+    section_content = itertools.takewhile(
+        lambda l: l.startswith("FAILED"), up_to_section_content
+    )
+    content = "\n".join(section_content)
+
+    return content
+
+
+def format_log_message(path):
+    py_version = path.name.split("-")[1]
+    summary = f"Python {py_version} Test Summary Info"
+    with open(path) as f:
+        data = extract_short_test_summary_info(line.rstrip() for line in f)
+    message = textwrap.dedent(
+        f"""\
+        <details><summary>{summary}</summary>
+
+        ```
+        {data}
+        ```
+
+        </details>
+        """
+    )
+
+    return message
 
-message = "\n"
 
 print("Parsing logs ...")
-for file in files:
-    with open(file) as fpt:
-        print(f"Parsing {file.absolute()}")
-        data = fpt.read().split("test summary info")[-1].splitlines()[1:-1]
-        data = "\n".join(data)
-        py_version = file.name.split("-")[1]
-        message = f"{message}\n<details>\n<summary>\nPython {py_version} Test Summary Info\n</summary>\n\n```bash\n{data}\n```\n</details>\n"
+message = "\n\n".join(format_log_message(path) for path in filepaths)
 
 output_file = pathlib.Path("pytest-logs.txt")
-with open(output_file, "w") as fpt:
-    print(f"Writing output file to: {output_file.absolute()} ")
-    fpt.write(message)
+print(f"Writing output file to: {output_file.absolute()}")
+output_file.write_text(message)

.github/workflows/upstream-dev-ci.yaml

@@ -21,6 +21,8 @@ jobs:
       fail-fast: false
       matrix:
         python-version: ["3.8"]
+    outputs:
+      artifacts_availability: ${{ steps.status.outputs.ARTIFACTS_AVAILABLE }}
     steps:
       - name: Cancel previous runs
         uses: styfle/[email protected]
@@ -30,6 +32,7 @@ jobs:
       - uses: conda-incubator/setup-miniconda@v2
         with:
           channels: conda-forge
+          channel-priority: strict
           mamba-version: "*"
           activate-environment: xarray-tests
           auto-update-conda: false
@@ -40,11 +43,17 @@ jobs:
           bash ci/install-upstream-wheels.sh
           conda list
       - name: Run Tests
+        id: status
         run: |
-          python -m pytest --verbose -rf > output-${{ matrix.python-version }}-log
-
+          set -euo pipefail
+          python -m pytest -rf | tee output-${{ matrix.python-version }}-log || (
+              echo '::set-output name=ARTIFACTS_AVAILABLE::true' && false
+          )
       - name: Upload artifacts
-        if: "failure()&&(github.event_name == 'schedule')&&(github.repository == 'pydata/xarray')" # Check the exit code of previous step
+        if: |
+          failure()
+          && github.event_name == 'schedule'
+          && github.repository == 'pydata/xarray'
         uses: actions/upload-artifact@v2
         with:
           name: output-${{ matrix.python-version }}-log
@@ -54,7 +63,11 @@ jobs:
   report:
     name: report
     needs: upstream-dev
-    if: "always()&&(github.event_name == 'schedule')&&(github.repository == 'pydata/xarray')"
+    if: |
+      always()
+      && github.event_name == 'schedule'
+      && github.repository == 'pydata/xarray'
+      && needs.upstream-dev.outputs.artifacts_availability == 'true'
     runs-on: ubuntu-latest
     defaults:
       run:
@@ -73,7 +86,8 @@ jobs:
           ls -R ./logs
       - name: Parse logs
         run: |
-          python .github/workflows/parse_logs.py
+          shopt -s globstar
+          python .github/workflows/parse_logs.py logs/**/*-log
       - name: Report failures
         uses: actions/github-script@v3
         with:
@@ -84,7 +98,7 @@ jobs:
             const title = "⚠️ Nightly upstream-dev CI failed ⚠️"
             const workflow_url = `https://github.com/${process.env.GITHUB_REPOSITORY}/actions/runs/${process.env.GITHUB_RUN_ID}`
             const issue_body = `[Workflow Run URL](${workflow_url})\n${pytest_logs}`
-            
+
             // Run GraphQL query against GitHub API to find the most recent open issue used for reporting failures
             const query = `query($owner:String!, $name:String!, $creator:String!, $label:String!){
               repository(owner: $owner, name: $name) {
@@ -107,11 +121,10 @@ jobs:
                 creator: "github-actions[bot]"
             }
             const result = await github.graphql(query, variables)
-            const issue_info = result.repository.issues.edges[0].node
 
-            // If no issue is open, create a new issue, else update the 
-            // body of the existing issue. 
-            if (typeof issue_info.number === 'undefined') {
+            // If no issue is open, create a new issue, 
+            // else update the body of the existing issue. 
+            if (result.repository.issues.edges.length === 0) {
                 github.issues.create({
                     owner: variables.owner,
                     repo: variables.name,
@@ -123,7 +136,7 @@ jobs:
                 github.issues.update({
                     owner: variables.owner,
                     repo: variables.name,
-                    issue_number: issue_info.number,
+                    issue_number: result.repository.issues.edges[0].node.number,
                     body: issue_body
                 })
             }

.pre-commit-config.yaml

@@ -11,7 +11,7 @@ repos:
     hooks:
       - id: black
   - repo: https://github.com/keewis/blackdoc
-    rev: v0.3
+    rev: v0.3.1
     hooks:
       - id: blackdoc
   - repo: https://gitlab.com/pycqa/flake8

ci/requirements/doc.yml

@@ -22,7 +22,7 @@ dependencies:
   - rasterio>=1.1
   - seaborn
   - setuptools
-  - sphinx=3.2
+  - sphinx=3.3
   - sphinx_rtd_theme>=0.4
   - sphinx-autosummary-accessors
   - zarr>=2.4

ci/requirements/py38.yml

@@ -29,7 +29,7 @@ dependencies:
   - numpy
   - pandas
   - pint
-  - pip
+  - pip=20.2
   - pseudonetcdf
   - pydap
   # - pynio: not compatible with netCDF4>1.5.3; only tested in py36-bare-minimum

doc/api-hidden.rst

@@ -689,6 +689,7 @@
    CFTimeIndex.dayofyear
    CFTimeIndex.dtype
    CFTimeIndex.empty
+   CFTimeIndex.freq
    CFTimeIndex.has_duplicates
    CFTimeIndex.hasnans
    CFTimeIndex.hour

doc/contributing.rst

@@ -51,8 +51,9 @@ Bug reports must:
    <http://github.github.com/github-flavored-markdown/>`_::
 
       ```python
-      >>> import xarray as xr
-      >>> df = xr.Dataset(...)
+      import xarray as xr
+      df = xr.Dataset(...)
+
       ...
       ```
 

doc/io.rst

@@ -1033,6 +1033,7 @@ and then calling ``to_zarr`` with ``compute=False`` to write only metadata
 .. ipython:: python
 
     import dask.array
+
     # The values of this dask array are entirely irrelevant; only the dtype,
     # shape and chunks are used
     dummies = dask.array.zeros(30, chunks=10)

doc/whats-new.rst

@@ -15,14 +15,45 @@ What's New
     np.random.seed(123456)
 
 
-.. _whats-new.0.16.2:
+.. _whats-new.{0.16.3}:
 
-v0.16.2 (unreleased)
---------------------
+v{0.16.3} (unreleased)
+----------------------
 
 Breaking changes
 ~~~~~~~~~~~~~~~~
 
+
+New Features
+~~~~~~~~~~~~
+
+
+Bug fixes
+~~~~~~~~~
+
+- :py:func:`merge` with ``combine_attrs='override'`` makes a copy of the attrs (:issue:`4627`).
+
+Documentation
+~~~~~~~~~~~~~
+
+
+Internal Changes
+~~~~~~~~~~~~~~~~
+
+
+.. _whats-new.0.16.2:
+
+v0.16.2 (30 Nov 2020)
+---------------------
+
+This release brings the ability to write to limited regions of ``zarr`` files, open zarr files with :py:func:`open_dataset` and :py:func:`open_mfdataset`, increased support for propagating ``attrs`` using the ``keep_attrs`` flag, as well as numerous bugfixes and documentation improvements.
+
+Many thanks to the 31 contributors who contributed to this release:
+Aaron Spring, Akio Taniguchi, Aleksandar Jelenak, alexamici, Alexandre Poux, Anderson Banihirwe, Andrew Pauling, Ashwin Vishnu, aurghs, Brian Ward, Caleb, crusaderky, Dan Nowacki, darikg, David Brochart, David Huard, Deepak Cherian, Dion Häfner, Gerardo Rivera, Gerrit Holl, Illviljan, inakleinbottle, Jacob Tomlinson, James A. Bednar, jenssss, Joe Hamman, johnomotani, Joris Van den Bossche, Julia Kent, Julius Busecke, Kai Mühlbauer, keewis, Keisuke Fujii, Kyle Cranmer, Luke Volpatti, Mathias Hauser, Maximilian Roos, Michaël Defferrard, Michal Baumgartner, Nick R. Papior, Pascal Bourgault, Peter Hausamann, PGijsbers, Ray Bell, Romain Martinez, rpgoldman, Russell Manser, Sahid Velji, Samnan Rahee, Sander, Spencer Clark, Stephan Hoyer, Thomas Zilio, Tobias Kölling, Tom Augspurger, Wei Ji, Yash Saboo, Zeb Nicholls,
+
+Deprecations
+~~~~~~~~~~~~
+
 - :py:attr:`~core.accessor_dt.DatetimeAccessor.weekofyear` and :py:attr:`~core.accessor_dt.DatetimeAccessor.week`
   have been deprecated. Use ``DataArray.dt.isocalendar().week``
   instead (:pull:`4534`). By `Mathias Hauser <https://github.com/mathause>`_,
@@ -55,24 +86,27 @@ New Features
   By `Julius Busecke <https://github.com/jbusecke>`_.
 - Added the ``keep_attrs`` keyword to ``rolling_exp.mean()``; it now keeps attributes
   per default. By `Mathias Hauser <https://github.com/mathause>`_ (:pull:`4592`).
+- Added ``freq`` as property to :py:class:`CFTimeIndex` and into the
+  ``CFTimeIndex.repr``. (:issue:`2416`, :pull:`4597`)
+  By `Aaron Spring <https://github.com/aaronspring>`_.
 
 Bug fixes
 ~~~~~~~~~
 
-- Fix bug where reference times without padded years (e.g. "since 1-1-1") would lose their units when
+- Fix bug where reference times without padded years (e.g. ``since 1-1-1``) would lose their units when
   being passed by :py:func:`encode_cf_datetime` (:issue:`4422`, :pull:`4506`). Such units are ambiguous
   about which digit represents the years (is it YMD or DMY?). Now, if such formatting is encountered,
-  it is assumed that the first digit is the years, they are padded appropriately (to e.g. "since 0001-1-1")
+  it is assumed that the first digit is the years, they are padded appropriately (to e.g. ``since 0001-1-1``)
   and a warning that this assumption is being made is issued. Previously, without ``cftime``, such times
   would be silently parsed incorrectly (at least based on the CF conventions) e.g. "since 1-1-1" would
-  be parsed (via``pandas`` and ``dateutil``) to "since 2001-1-1".
+  be parsed (via ``pandas`` and ``dateutil``) to ``since 2001-1-1``.
   By `Zeb Nicholls <https://github.com/znicholls>`_.
 - Fix :py:meth:`DataArray.plot.step`. By `Deepak Cherian <https://github.com/dcherian>`_.
 - Fix bug where reading a scalar value from a NetCDF file opened with the ``h5netcdf`` backend would raise a ``ValueError`` when ``decode_cf=True`` (:issue:`4471`, :pull:`4485`).
   By `Gerrit Holl <https://github.com/gerritholl>`_.
 - Fix bug where datetime64 times are silently changed to incorrect values if they are outside the valid date range for ns precision when provided in some other units (:issue:`4427`, :pull:`4454`).
   By `Andrew Pauling <https://github.com/andrewpauling>`_
-- Fix silently overwriting the `engine` key when passing :py:func:`open_dataset` a file object
+- Fix silently overwriting the ``engine`` key when passing :py:func:`open_dataset` a file object
   to an incompatible netCDF (:issue:`4457`). Now incompatible combinations of files and engines raise
   an exception instead. By `Alessandro Amici <https://github.com/alexamici>`_.
 - The ``min_count`` argument to :py:meth:`DataArray.sum()` and :py:meth:`DataArray.prod()`
@@ -91,7 +125,9 @@ Documentation
 ~~~~~~~~~~~~~
 - document the API not supported with duck arrays (:pull:`4530`).
   By `Justus Magin <https://github.com/keewis>`_.
-
+- Mention the possibility to pass functions to :py:meth:`Dataset.where` or
+  :py:meth:`DataArray.where` in the parameter documentation (:issue:`4223`, :pull:`4613`).
+  By `Justus Magin <https://github.com/keewis>`_.
 - Update the docstring of :py:class:`DataArray` and :py:class:`Dataset`.
   (:pull:`4532`);
   By `Jimmy Westling <https://github.com/illviljan>`_.
@@ -104,6 +140,9 @@ Documentation
   By `Sahid Velji <https://github.com/sahidvelji>`_.
 - Update link to NumPy docstring standard in the :doc:`contributing` guide (:pull:`4558`).
   By `Sahid Velji <https://github.com/sahidvelji>`_.
+- Add docstrings to ``isnull`` and ``notnull``, and fix the displayed signature
+  (:issue:`2760`, :pull:`4618`).
+  By `Justus Magin <https://github.com/keewis>`_.
 
 Internal Changes
 ~~~~~~~~~~~~~~~~
@@ -118,7 +157,7 @@ Internal Changes
   <https://github.com/mathause>`_.
 - Removed stray spaces that stem from black removing new lines (:pull:`4504`).
   By `Mathias Hauser <https://github.com/mathause>`_.
-- Ensure tests are not skipped in the `py38-all-but-dask` test environment
+- Ensure tests are not skipped in the ``py38-all-but-dask`` test environment
   (:issue:`4509`). By `Mathias Hauser <https://github.com/mathause>`_.
 - Ignore select numpy warnings around missing values, where xarray handles
   the values appropriately, (:pull:`4536`);
@@ -128,6 +167,10 @@ Internal Changes
   (:issue:`4565`). By `Mathias Hauser <https://github.com/mathause>`_.
 - Add GitHub action for running nightly tests against upstream dependencies (:pull:`4583`). 
   By `Anderson Banihirwe <https://github.com/andersy005>`_. 
+- Ensure all figures are closed properly in plot tests (:pull:`4600`).
+  By `Yash Saboo <https://github.com/yashsaboo>`_, `Nirupam K N
+  <https://github.com/Nirupamkn>`_ and `Mathias Hauser
+  <https://github.com/mathause>`_.
 
 .. _whats-new.0.16.1:
 

xarray/backends/api.py

@@ -26,7 +26,7 @@
     combine_by_coords,
 )
 from ..core.dataarray import DataArray
-from ..core.dataset import Dataset, _maybe_chunk
+from ..core.dataset import Dataset, _get_chunk, _maybe_chunk
 from ..core.utils import close_on_error, is_grib_path, is_remote_uri
 from .common import AbstractDataStore, ArrayWriter
 from .locks import _get_scheduler
@@ -536,7 +536,7 @@ def maybe_decode_store(store, chunks):
                 k: _maybe_chunk(
                     k,
                     v,
-                    store.get_chunk(k, v, chunks),
+                    _get_chunk(k, v, chunks),
                     overwrite_encoded_chunks=overwrite_encoded_chunks,
                 )
                 for k, v in ds.variables.items()