Add requirements regenerator script and action

Author: timkpaine

.github/workflows/build.yml

@@ -309,6 +309,8 @@ jobs:
             # Python
             - name: Install python dependencies
               run: yarn _requires_python
+              env:
+                  PSP_FROZEN_REQUIREMENTS: "1"
 
             ################
             # Linux
@@ -508,6 +510,8 @@ jobs:
             # Python
             - name: Install python dependencies
               run: yarn _requires_python
+              env:
+                  PSP_FROZEN_REQUIREMENTS: "1"
 
             #~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~#
             #~~~~~~~~~ Build Pipelines ~~~~~~~~~#
@@ -641,6 +645,8 @@ jobs:
             # Python
             - name: Install python dependencies
               run: yarn _requires_python
+              env:
+                  PSP_FROZEN_REQUIREMENTS: "1"
 
             ################
             # Rust
@@ -1029,6 +1035,8 @@ jobs:
             - name: Install python dependencies
               run: yarn _requires_python
               if: ${{ runner.os != 'Linux' }} # skip on manylinux2014
+              env:
+                  PSP_FROZEN_REQUIREMENTS: "1"
 
             ################
             # Linux
@@ -1323,6 +1331,8 @@ jobs:
             # Python
             - name: Install python dependencies
               run: yarn _requires_python
+              env:
+                  PSP_FROZEN_REQUIREMENTS: "1"
 
             # Download artifact
             - uses: actions/download-artifact@v3
@@ -1491,6 +1501,8 @@ jobs:
             # Python
             - name: Install python dependencies
               run: yarn _requires_python
+              env:
+                  PSP_FROZEN_REQUIREMENTS: "1"
 
             #~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~#
             #~~~~~~~~~ Build Pipelines ~~~~~~~~~#
@@ -1636,6 +1648,8 @@ jobs:
             # Python
             - name: Install python dependencies
               run: yarn _requires_python
+              env:
+                  PSP_FROZEN_REQUIREMENTS: "1"
 
             #~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~#
             #~~~~~~~~~ Build Pipelines ~~~~~~~~~#
@@ -1751,6 +1765,8 @@ jobs:
             # Python
             - name: Install python dependencies
               run: yarn _requires_python
+              env:
+                  PSP_FROZEN_REQUIREMENTS: "1"
 
             #~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~#
             #~~~~~~~~~ Build Pipelines ~~~~~~~~~#

.github/workflows/python-deps.yml

@@ -0,0 +1,62 @@
+name: Regenerate Python Dependencies
+
+on:
+    workflow_dispatch:
+
+concurrency:
+    group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
+    cancel-in-progress: true
+
+permissions:
+    contents: write
+
+jobs:
+    generate:
+        strategy:
+            matrix:
+                python-version:
+                    - "3.7"
+                    - "3.8"
+                    - "3.9"
+                    - "3.10"
+                    - "3.11"
+                node-version: [18.x]
+
+            # NOTE: don't fail fast as sometimes npm blocks the burst of fetches from GHA
+            fail-fast: false
+
+        runs-on: ubuntu-20.04
+
+        steps:
+            - name: Checkout
+              uses: actions/checkout@v3
+
+            - name: Set up Python ${{ matrix.python-version }}
+              uses: actions/setup-python@v4
+              with:
+                  python-version: ${{ matrix.python-version }}
+                  cache: "pip"
+
+            - name: Use Node.js ${{ matrix.node-version }}
+              uses: actions/setup-node@v3
+              with:
+                  node-version: ${{ matrix.node-version }}
+                  cache: "yarn"
+                  cache-dependency-path: yarn.lock
+
+            - name: Install js dependencies
+              run: yarn
+              env:
+                  PSP_SKIP_EMSDK_INSTALL: 1
+
+            - name: Install python dependencies
+              run: yarn _requires_python
+              env:
+                  PSP_GENERATE_FROZEN_REQUIREMENTS: "1"
+
+            - name: Push changes to well-known branch name
+              run: |
+                  git checkout -b tkp/update-frozen-python-dependencies
+                  git add python/perspective/ci-requirements/
+                  git commit --author="3105306+timkpaine@users.noreply.github.com" -m "Update frozen python dependencies"
+                  git push origin tkp/update-frozen-python-dependencies -f

.gitignore

@@ -132,6 +132,9 @@ website/i18n/*
 !website/i18n/en.json
 website/static/css/material-dark.css
 website/static/css/pro-dark.css
+docs/static/features
+docs/.docusaurus
+docs/static/blocks
 
 # test artifacts
 .ipynb_checkpoints
@@ -143,6 +146,10 @@ coverage
 screenshots/
 junit.xml
 results.debug.json
+test-results/
+playwright-report/
+playwright/.cache/
+.cache
 
 # CPP Compile
 /src/include/boost
@@ -204,21 +211,10 @@ vcpkg
 # jupyterlab test artifacts
 packages/perspective-jupyterlab/test/config/jupyter/lab
 packages/perspective-jupyterlab/test/config/jupyter/migrated
-docs/static/features
-results.debug.json
-
-tools/perspective-build/lib
-docs/.docusaurus
-packages/perspective-esbuild-plugin/lib
-docs/static/blocks
-.vscode/c_cpp_properties.json
-test-results/
-playwright-report/
-playwright/.cache/
 
+# Build envs / virtualenvs / other local
 .pyodide-xbuildenv
 benchmark_venv
-
 venv/
+.venvs/
 testenv
-.cache

python/perspective/MANIFEST.in

@@ -31,6 +31,9 @@ prune bench
 # Don't ship scripts
 prune scripts
 
+# Don't ship test artifacts
+prune ci-requirements
+
 # Patterns to exclude from any directory
 global-exclude *~
 global-exclude *.pyc

python/perspective/requirements-310.txt …ive/ci-requirements/requirements-310.txtpython/perspective/requirements-310.txt renamed to python/perspective/ci-requirements/requirements-310.txt python/perspective/requirements-310.txt renamed to python/perspective/ci-requirements/requirements-310.txt

@@ -13,28 +13,17 @@
 import {
     default as sh,
     python_version,
-    copy_files_to_python_folder,
-    py_requirements,
+    generate_py_requirements,
+    install_py_requirements_dynamic,
+    install_py_requirements_frozen,
 } from "./sh_perspective.mjs";
 
 let PYTHON = sh(python_version());
 
-if (process.env.PSP_OLD_SHITTY_INSTALL_METHOD) {
-    const requires_script = `import distutils.core; setup = distutils.core.run_setup('python/perspective/setup.py'); print(' '.join(['"' + requirement + '"' for requirement in setup.extras_require['dev']]))`;
-
-    // copy build/config files into python folder
-    copy_files_to_python_folder();
-
-    // install build meta  deps, this is a necessary evil to keep the setup.py clean
-    sh`${PYTHON} -m pip install  jupyter_packaging==0.12.3`.runSync();
-    const requirements = await sh`${PYTHON} -c ${requires_script}`.execSync();
-    if (requirements.trim().length > 0) {
-        console.log(`Installing: ${requirements}`);
-        sh`${PYTHON} -m pip install -U ${sh(requirements)}`.log().runSync();
-    } else {
-        console.log("Nothing to install");
-    }
+if (process.env.PSP_GENERATE_FROZEN_REQUIREMENTS) {
+    generate_py_requirements(PYTHON);
+} else if (process.env.PSP_FROZEN_REQUIREMENTS) {
+    install_py_requirements_frozen(PYTHON);
 } else {
-    sh`${PYTHON} -m pip install -r ${py_requirements()}`.runSync();
-    sh`${PYTHON} -m pip install -U jupyter_packaging==0.12.3`.runSync();
+    install_py_requirements_dynamic(PYTHON);
 }