Merge branch 'main' into main

Author: zanieb

.coveragerc

@@ -1,6 +1,6 @@
 [run]
-include = 
-    distributed/*
+source =
+    distributed
 omit =
     distributed/tests/test*
     distributed/hdfs.py

.github/PULL_REQUEST_TEMPLATE.md

@@ -1,3 +1,3 @@
 - [ ] Closes #xxxx
 - [ ] Tests added / passed
-- [ ] Passes `black distributed` / `flake8 distributed` / `isort distributed`
+- [ ] Passes `pre-commit run --all-files`

.github/workflows/tests.yaml

@@ -116,12 +116,15 @@ jobs:
           source continuous_integration/scripts/set_ulimit.sh
 
           pytest distributed -m "not avoid_ci and ${{ matrix.partition }}" --runslow \
-            --junitxml reports/pytest.xml -o junit_suite_name=$TEST_ID
+            --junitxml reports/pytest.xml -o junit_suite_name=$TEST_ID --cov=distributed --cov-report=xml
 
       # - name: Debug with tmate on failure
       #   if: ${{ failure() }}
       #   uses: mxschmitt/action-tmate@v3
 
+      - name: Coverage
+        uses: codecov/codecov-action@v1
+
       - name: Upload test artifacts
         # ensure this runs even if pytest fails
         if: >

.gitignore

@@ -1,8 +1,12 @@
 *.pyc
 *.pyd
 *.py~
+
+# Cython files
 *.c
 *.so
+distributed/scheduler.html
+
 build/
 dist/
 *.egg-info
@@ -11,6 +15,8 @@ continuous_integration/hdfs-initialized
 .DS_Store
 .cache
 .coverage
+.coverage.*
+coverage.xml
 *.lock
 .#*
 .eggs/
@@ -23,3 +29,4 @@ dask-worker-space/
 .ycm_extra_conf.py
 tags
 .ipynb_checkpoints
+.venv/

.pre-commit-config.yaml

@@ -1,11 +1,19 @@
 repos:
   -   repo: https://github.com/pycqa/isort
-      rev: 5.8.0
+      rev: 5.9.3
       hooks:
       - id: isort
         language_version: python3
+  - repo: https://github.com/asottile/pyupgrade
+    # Do not upgrade: there's a bug in Cython that causes sum(... for ...) to fail;
+    # it needs sum([... for ...])
+    rev: v2.13.0
+    hooks:
+      - id: pyupgrade
+        args:
+          - --py37-plus
   -   repo: https://github.com/psf/black
-      rev: 21.5b1
+      rev: 21.9b0
       hooks:
       - id: black
         language_version: python3
@@ -17,3 +25,22 @@ repos:
       hooks:
       - id: flake8
         language_version: python3
+  - repo: https://github.com/pre-commit/mirrors-mypy
+    rev: v0.910
+    hooks:
+      - id: mypy
+        additional_dependencies:
+          # Type stubs
+          - types-docutils
+          - types-requests
+          - types-paramiko
+          - types-pkg_resources
+          - types-PyYAML
+          - types-setuptools
+          - types-psutil
+          # Libraries exclusively imported under `if TYPE_CHECKING:`
+          - typing_extensions  # To be reviewed after dropping Python 3.7
+          # Typed libraries
+          - numpy
+          - dask
+          - tornado

README.rst

@@ -1,7 +1,7 @@
 Distributed
 ===========
 
-|Test Status| |Doc Status| |Gitter| |Version Status| |NumFOCUS|
+|Test Status| |Coverage| |Doc Status| |Gitter| |Version Status| |NumFOCUS|
 
 A library for distributed computation.  See documentation_ for more details.
 
@@ -11,6 +11,9 @@ A library for distributed computation.  See documentation_ for more details.
 .. |Doc Status| image:: https://readthedocs.org/projects/distributed/badge/?version=latest
    :target: https://distributed.dask.org
    :alt: Documentation Status
+.. |Coverage| image:: https://codecov.io/gh/dask/distributed/branch/main/graph/badge.svg
+   :target: https://codecov.io/gh/dask/distributed/branch/main
+   :alt: Coverage status
 .. |Gitter| image:: https://badges.gitter.im/Join%20Chat.svg
    :alt: Join the chat at https://gitter.im/dask/dask
    :target: https://gitter.im/dask/dask?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge

codecov.yml

@@ -0,0 +1,17 @@
+codecov:
+  require_ci_to_pass: yes
+
+coverage:
+  precision: 2
+  round: down
+  range: "85...100"
+
+  status:
+    project:
+      default:
+        target: 85%
+        threshold: 1%
+    patch: no
+    changes: no
+
+comment: off

continuous_integration/environment-3.7.yaml

@@ -9,6 +9,7 @@ dependencies:
   - bokeh
   - click
   - cloudpickle
+  - coverage
   - cython  # Only tested here; also a dependency of crick
   - dask  # overridden by git tip below
   - filesystem-spec
@@ -17,15 +18,17 @@ dependencies:
   - ipywidgets
   - jinja2
   - joblib
-  - jupyter_client<7  # FIXME distributed#5272
+  - jupyter_client
   - msgpack-python
   - netcdf4
   - paramiko
   - pickle5  # Backport from Python 3.8
+  - pre-commit
   - prometheus_client
   - psutil
   - pytest
   - pytest-asyncio<0.14.0
+  - pytest-cov
   - pytest-faulthandler
   - pytest-repeat
   - pytest-rerunfailures

continuous_integration/environment-3.8.yaml

@@ -9,21 +9,24 @@ dependencies:
   - bokeh
   - click
   - cloudpickle
+  - coverage
   - dask  # overridden by git tip below
   - filesystem-spec
   - h5py
   - ipykernel
   - ipywidgets
   - jinja2
   - joblib
-  - jupyter_client<7  # FIXME distributed#5272
+  - jupyter_client
   - msgpack-python
   - netcdf4
   - paramiko
+  - pre-commit
   - prometheus_client
   - psutil
   - pytest
   - pytest-asyncio<0.14.0
+  - pytest-cov
   - pytest-faulthandler
   - pytest-repeat
   - pytest-rerunfailures

continuous_integration/environment-3.9.yaml

@@ -10,23 +10,26 @@ dependencies:
   - bokeh
   - click
   - cloudpickle
+  - coverage
   - dask  # overridden by git tip below
   - filesystem-spec  # overridden by git tip below
   - h5py
   - ipykernel
   - ipywidgets
   - jinja2
   - joblib  # overridden by git tip below
-  - jupyter_client<7  # FIXME distributed#5272
+  - jupyter_client
   - lz4  # Only tested here
   - msgpack-python
   - netcdf4
   - paramiko
+  - pre-commit
   - prometheus_client
   - psutil
   - pynvml  # Only tested here
   - pytest
   - pytest-asyncio<0.14.0
+  - pytest-cov
   - pytest-faulthandler
   - pytest-repeat
   - pytest-rerunfailures
@@ -49,6 +52,7 @@ dependencies:
       - git+https://github.com/dask/dask
       - git+https://github.com/dask/s3fs
       - git+https://github.com/dask/zict
-      - git+https://github.com/intake/filesystem_spec
+      # FIXME https://github.com/dask/distributed/issues/5345
+      # - git+https://github.com/intake/filesystem_spec
       - git+https://github.com/joblib/joblib
       - keras