Cleanup up python requirements installation in CI, remove some dangling azure references, refresh python README, add conda badge

Author: timkpaine

.artifactignore

@@ -304,7 +304,7 @@ jobs:
     ################
     # Python
     - name: Install python dependencies
-      run: python -m pip install --upgrade "jupyterlab==3.4.3" "tornado==6.1"
+      run: yarn _requires_python
 
     ################
     # Linux
@@ -490,7 +490,7 @@ jobs:
     ################
     # Python
     - name: Install python dependencies
-      run: python -m pip install --upgrade "black==20.8b1" flake8-black
+      run: yarn _requires_python
 
     #~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~#
     #~~~~~~~~~ Build Pipelines ~~~~~~~~~#
@@ -618,7 +618,7 @@ jobs:
     ################
     # Python
     - name: Install python dependencies
-      run: python -m pip install --upgrade "jupyterlab==3.4.3" "tornado==6.1"
+      run: yarn _requires_python
 
     ################
     # Wasm
@@ -910,7 +910,7 @@ jobs:
     ################
     # Python
     - name: Install python dependencies
-      run: python -m pip install --upgrade pip wheel setuptools "jupyterlab==3.4.3" numpy "pyarrow>=5" "tornado==6.1"
+      run: yarn _requires_python
       if: ${{ runner.os != 'Linux' }}  # skip on manylinux2014
 
     ################
@@ -1161,7 +1161,7 @@ jobs:
     ################
     # Python
     - name: Install python dependencies
-      run: python -m pip install --upgrade "jupyterlab==3.4.3" ipywidgets ipykernel "tornado==6.1"
+      run: yarn _requires_python
 
     # Download artifact
     - uses: actions/download-artifact@v3
@@ -1330,7 +1330,7 @@ jobs:
     ################
     # Python
     - name: Install python dependencies
-      run: python -m pip install --upgrade aiohttp Faker fastapi "jupyterlab==3.4.3" mock numpy pip psutil "pyarrow>=5" pytest pytest-cov pytest-aiohttp pytest-asyncio pytest-tornado pytz setuptools wheel "tornado==6.1"
+      run: yarn _requires_python
 
     #~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~#
     #~~~~~~~~~ Build Pipelines ~~~~~~~~~#
@@ -1466,7 +1466,7 @@ jobs:
     ################
     # Python
     - name: Install python dependencies
-      run: python -m pip install --upgrade pip wheel setuptools "jupyterlab==3.4.3" numpy "pyarrow>=5" pytest pytest-cov mock aiohttp fastapi Faker psutil pytest-aiohttp pytest-asyncio pytest-tornado pytz "tornado==6.1"
+      run: yarn _requires_python
 
     #~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~#
     #~~~~~~~~~ Build Pipelines ~~~~~~~~~#
@@ -1482,7 +1482,7 @@ jobs:
 
     # Test sdist 
     - name: Run tests against from-scratch sdist build
-      run: python -m pytest -vvv --asyncio-mode=strict `python3 -c "import sysconfig; print(sysconfig.get_path('purelib'))"`/perspective/tests --ignore=`python3 -c "import sysconfig; print(sysconfig.get_path('purelib'))"`/perspective/tests/client_mode
+      run: python -m pytest -vvv --rootdir=python/perspective `python3 -c "import sysconfig; print(sysconfig.get_path('purelib'))"`/perspective/tests --ignore=`python3 -c "import sysconfig; print(sysconfig.get_path('purelib'))"`/perspective/tests/client_mode
 
   ##########################################################################################################################
   ##########################################################################################################################

.github/workflows/build.yml

@@ -3,6 +3,7 @@
 
 [![npm](https://img.shields.io/npm/v/@finos/perspective.svg?style=flat)](https://www.npmjs.com/package/@finos/perspective)
 [![PyPI](https://img.shields.io/pypi/v/perspective-python.svg?style=flat)](https://pypi.python.org/pypi/perspective-python)
+![Conda](https://img.shields.io/conda/v/conda-forge/perspective)
 [![Build Status](https://github.com/finos/perspective/actions/workflows/build.yml/badge.svg?branch=master&event=push)](https://github.com/finos/perspective/actions/workflows/build.yml)
 
 Perspective is an <i>interactive</i> analytics and data visualization component,

README.md

@@ -117,7 +117,6 @@
         "build_js": "node scripts/build_js.js",
         "build": "node scripts/build.js",
         "build_python": "node scripts/build_python.js",
-        "build_python_azure": "node scripts/build_python_azure.js",
         "bench": "node scripts/bench.js",
         "bench_python": "python3 python/perspective/bench/perspective_benchmark.py",
         "ci:full": "git commit --allow-empty -m \"[ci-full]\"",
@@ -126,6 +125,7 @@
         "ci:skip-cache": "git commit --allow-empty -m \"[ci-skip-cache]\"",
         "stresstest_python": "python3 python/perspective/bench/stresstest/main.py",
         "_wheel_python": "node scripts/_wheel_python.js",
+        "_requires_python": "node scripts/_requires_python.js",
         "setup": "node scripts/setup.js",
         "docs": "node scripts/docs.js",
         "test": "node scripts/test.js",

package.json

@@ -2,42 +2,50 @@
 
 Python APIs for [perspective](https://github.com/finos/perspective) front end
 
-[![npm](https://img.shields.io/npm/v/@finos/perspective.svg?style=flat-square)](https://www.npmjs.com/package/@finos/perspective)
-[![PyPI](https://img.shields.io/pypi/v/perspective-python.svg?style=flat-square)](https://pypi.python.org/pypi/perspective-python)
-[![Build Status](https://dev.azure.com/finosfoundation/perspective/_apis/build/status/finos.perspective?branchName=master)](https://dev.azure.com/finosfoundation/perspective/_build/latest?definitionId=1&branchName=master)
-[![FINOS - Active](https://cdn.jsdelivr.net/gh/finos/contrib-toolbox@master/images/badge-active.svg)](https://finosfoundation.atlassian.net/wiki/display/FINOS/Active)
+[![npm](https://img.shields.io/npm/v/@finos/perspective.svg?style=flat)](https://www.npmjs.com/package/@finos/perspective)
+[![PyPI](https://img.shields.io/pypi/v/perspective-python.svg?style=flat)](https://pypi.python.org/pypi/perspective-python)
+![Conda](https://img.shields.io/conda/v/conda-forge/perspective)
+[![Build Status](https://github.com/finos/perspective/actions/workflows/build.yml/badge.svg?branch=master&event=push)](https://github.com/finos/perspective/actions/workflows/build.yml)
 [![Binder](http://mybinder.org/badge_logo.svg)](http://beta.mybinder.org/v2/gh/finos/perspective/master?urlpath=lab/tree/examples/jupyter-notebooks)
 
-Perspective is an <i>interactive</i> visualization component for <i>large</i>, <i>real-time</i>
-datasets. Originally developed for J.P. Morgan's trading business,  Perspective
-makes it simple to build real-time & user configurable analytics entirely in the
-browser, or in concert with Python and/or
-[Jupyterlab](https://jupyterlab.readthedocs.io/en/stable/).
-Use it to create reports, dashboards, notebooks and applications, with static
-data or streaming updates via [Apache Arrow](https://arrow.apache.org/).
-
-<img src="https://perspective.finos.org/img/demo_small.gif">
+Perspective is an <i>interactive</i> analytics and data visualization component,
+which is especially well-suited for <i>large</i> and/or <i>streaming</i>
+datasets. Use it to create user-configurable reports, dashboards, notebooks and
+applications, then deploy stand-alone in the browser, or in concert with Python
+and/or [Jupyterlab](https://jupyterlab.readthedocs.io/en/stable/).
 
 ## Features
 
-- A fast, memory efficient streaming query engine, written in C++ and compiled to [WebAssembly](https://webassembly.org/), with read/write/stream support for [Apache Arrow](https://arrow.apache.org/).
-
-- A framework-agnostic query configuration UI component, based on [Web Components](https://www.webcomponents.org/), and a WebWorker and/or WebSocket data engine host for stable interactivity at high frequency.
-
-- A customizable HTML Data Grid plugin, and a Chart plugin built on [D3FC](https://d3fc.io/).
+-   A fast, memory efficient streaming query engine, written in
+    C++ and compiled for both [WebAssembly](https://webassembly.org/) and
+    [Python](https://www.python.org/), with read/write/streaming for
+    [Apache Arrow](https://arrow.apache.org/), and a high-performance columnar
+    expression language based on [ExprTK](https://github.com/ArashPartow/exprtk).
 
-- Integration with [Jupyterlab](https://jupyterlab.readthedocs.io/en/stable/), both natively in a Python kernel, and as a notebook Widget.
+-   A framework-agnostic User Interface packaged as a
+    [Custom Element](https://developer.mozilla.org/en-US/docs/Web/Web_Components/Using_custom_elements),
+    powered either in-browser via WebAssembly or virtually via
+    WebSocket server (Python/Node).
 
-- Cross-language streaming and/or virtualization to the browser via [Apache Arrow](https://arrow.apache.org/).
-
-- Runtimes for the browser, Python, and Node.js.
+-   A [JupyterLab](https://jupyter.org/) widget and Python client library, for
+    interactive data analysis in a notebook, as well as _scalable_ production
+    [Voila](https://github.com/voila-dashboards/voila) applications.
 
 ## Documentation
 
 * [Project Site](https://perspective.finos.org/)
-* [Python Installation and User Guide](https://perspective.finos.org/docs/md/python.html)
-* [Table (Conceptual Overview)](https://perspective.finos.org/docs/md/table.html)
-* [View (Conceptual Overview)](https://perspective.finos.org/docs/md/view.html)
+* [Table](https://perspective.finos.org/docs/md/table.html)
+* [View](https://perspective.finos.org/docs/md/view.html)
+* [Javascript User Guide](https://perspective.finos.org/docs/md/js.html)
+* [Python User Guide](https://perspective.finos.org/docs/md/python.html)
 * [Data Binding](https://perspective.finos.org/docs/md/table.html)
-* [Developer's Guide](https://perspective.finos.org/docs/md/development.html)
-* [Perspective Python API](https://perspective.finos.org/docs/obj/perspective-python.html)
+* [Developer Guide](https://perspective.finos.org/docs/md/development.html)
+* [Perspective API](https://github.com/finos/perspective/blob/master/packages/perspective/README.md)
+* [Perspective Viewer API](https://github.com/finos/perspective/blob/master/packages/perspective-viewer/README.md)
+* [Perspective Python API](https://perspective.finos.org/docs/obj/perspective-python.html)
+
+## Community
+
+* [`perspective-viewer-maps` OpenLayers/OpenStreetMap plugin](https://github.com/DevAndyLee/perspective-viewer-maps)
+* [Scott Logic / Perspective Plugin API - How to build a new plugin](https://blog.scottlogic.com/2019/04/23/perspective-plugin-api-how-to-build-a-new-plugin.html)
+* [Streaming, cross-sectional data visualization in Jupyterlab | Junyuan Tan, JupyterCon 2020](http://www.youtube.com/watch?v=IO-HJsGdleE)

python/perspective/README.md

@@ -62,9 +62,11 @@ def get_version(file, name="__version__"):
 
 requires_aiohttp = ["aiohttp"]
 
+requires_jupyter = ["jupyterlab>=3.2,<4"]
+
 requires_starlette = ["fastapi", "starlette"]
 
-requires_tornado = ["tornado>=4.5.3"]
+requires_tornado = ["tornado>=4.5.3,<6.2"]
 
 requires_dev = (
     [
@@ -89,6 +91,7 @@ def get_version(file, name="__version__"):
     ]
     + requires
     + requires_aiohttp
+    + requires_jupyter
     + requires_starlette
     + requires_tornado
 )
@@ -278,6 +281,7 @@ def run_check(self):
         "Programming Language :: Python :: 3.7",
         "Programming Language :: Python :: 3.8",
         "Programming Language :: Python :: 3.9",
+        "Programming Language :: Python :: 3.10",
     ],
     keywords="analytics tools plotting",
     packages=find_packages(exclude=["bench", "bench.*"]),
@@ -288,7 +292,9 @@ def run_check(self):
     extras_require={
         "aiohttp": requires_aiohttp,
         "dev": requires_dev,
+        "develop": requires_dev,
         "fastapi": requires_starlette,
+        "jupyter": requires_jupyter,
         "starlette": requires_starlette,
         "tornado": requires_tornado,
     },

python/perspective/setup.py

@@ -0,0 +1,29 @@
+/******************************************************************************
+ *
+ * Copyright (c) 2017, the Perspective Authors.
+ *
+ * This file is part of the Perspective library, distributed under the terms of
+ * the Apache License 2.0.  The full license can be found in the LICENSE file.
+ *
+ */
+const {execute, execute_return, python_version} = require("./script_utils.js");
+
+let PYTHON = python_version();
+const requires_script = `'import distutils.core; setup = distutils.core.run_setup("python/perspective/setup.py"); print(" ".join(["\\"" + requirement + "\\"" for requirement in setup.extras_require["dev"]]))'`;
+
+(async () => {
+    try {
+        const {stdout: requirements} =
+            await execute_return`${PYTHON} -c ${requires_script}`;
+
+        console.log(`Installing: ${requirements}`);
+
+        const cmd = `${PYTHON} -m pip install ${requirements}`;
+        console.log(cmd);
+
+        execute`${cmd}`;
+    } catch (e) {
+        console.log(e.message);
+        process.exit(1);
+    }
+})();