Merge pull request #1257 from github/rasmuswl/fix-ubuntu22.04-venv-creation

python-setup: Fix venv creation in Ubuntu 22.04

Author: henrymercer

.github/workflows/python-deps.yml

@@ -26,7 +26,7 @@ jobs:
     strategy:
         fail-fast: false
         matrix:
-          os: [ubuntu-latest, macos-latest]
+          os: [ubuntu-latest, ubuntu-22.04, macos-latest]
           python_deps_type: [pipenv, poetry, requirements, setup_py]
           python_version: [2, 3]
           exclude:
@@ -36,6 +36,9 @@ jobs:
             # Python2 and pipenv are not supported since pipenv v2021.11.5
             - python_version: 2
               python_deps_type: pipenv
+            # Python2 is not available on ubuntu-22.04 by default -- see https://github.com/github/codeql-action/pull/1257
+            - python_version: 2
+              os: ubuntu-22.04
 
 
     env:
@@ -63,6 +66,7 @@ jobs:
 
         case ${{ matrix.os }} in
             ubuntu-latest*)     basePath="/opt";;
+            ubuntu-22.04*)     basePath="/opt";;
             macos-latest*)    basePath="/Users/runner";;
         esac
         echo ${basePath}
@@ -86,7 +90,7 @@ jobs:
     strategy:
         fail-fast: false
         matrix:
-          os: [ubuntu-latest, macos-latest]
+          os: [ubuntu-latest, ubuntu-22.04, macos-latest]
 
     steps:
     # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
@@ -109,6 +113,7 @@ jobs:
 
         case ${{ matrix.os }} in
             ubuntu-latest*)     basePath="/opt";;
+            ubuntu-22.04*)     basePath="/opt";;
             macos-latest*)    basePath="/Users/runner";;
         esac
         echo ${basePath}

python-setup/auto_install_packages.py

@@ -5,6 +5,7 @@
 import subprocess
 from tempfile import mkdtemp
 from typing import Optional
+import shutil
 
 import extractor_version
 
@@ -167,6 +168,19 @@ def install_packages(codeql_base_dir) -> Optional[str]:
 
     # get_extractor_version returns the Python version the extractor thinks this repo is using
     version = extractor_version.get_extractor_version(codeql_base_dir, quiet=False)
+    sys.stdout.flush()
+    sys.stderr.flush()
+
+    if version == 2 and not sys.platform.startswith('win32'):
+        # On Ubuntu 22.04 'python2' is not available by default. We want to give a slightly better
+        # error message than a traceback + `No such file or directory: 'python2'`
+        if shutil.which("python2") is None:
+            sys.exit(
+                "Python package installation failed: we detected this code as Python 2, but the 'python2' executable was not available. "
+                "To enable automatic package installation, please install 'python2' before the 'github/codeql-action/init' step, "
+                "for example by running 'sudo apt install python2' (Ubuntu 22.04). "
+                "If your code is not Python 2, but actually Python 3, please file a bug report at https://github.com/github/codeql-action/issues/new"
+            )
 
     if os.path.exists('requirements.txt'):
         print('Found requirements.txt, will install packages with pip', flush=True)

python-setup/install_tools.ps1

@@ -1,7 +1,11 @@
 #! /usr/bin/pwsh
 
-py -2 -m pip install --user --upgrade pip setuptools wheel
-py -3 -m pip install --user --upgrade pip setuptools wheel
+# while waiting for the next release of `virtualenv` after v20.16.5, we install an older
+# version of `setuptools` to ensure that binaries are always put under
+# `<venv-path>/bin`, which wouldn't always happen with the GitHub actions version of
+# Ubuntu 22.04. See https://github.com/github/codeql-action/issues/1249
+py -2 -m pip install --user --upgrade pip 'setuptools<60' wheel
+py -3 -m pip install --user --upgrade pip 'setuptools<60' wheel
 
 # virtualenv is a bit nicer for setting up virtual environment, since it will provide up-to-date versions of
 # pip/setuptools/wheel which basic `python3 -m venv venv` won't

python-setup/install_tools.sh

@@ -11,7 +11,13 @@ set -e
 export PATH="$HOME/.local/bin:$PATH"
 
 # Setup Python 3 dependency installation tools.
-python3 -m pip install --user --upgrade pip setuptools wheel
+
+# we install an older version of `setuptools` to ensure that binaries are always put
+# under `<venv-path>/bin`, which wouldn't always happen with the GitHub actions version
+# of Ubuntu 22.04. See https://github.com/github/codeql-action/issues/1249. The the next
+# release of `virtualenv` after v20.16.5 will include a fix for this, so we can remove
+# this bit of the logic again.
+python3 -m pip install --user --upgrade pip 'setuptools<60' wheel
 
 # virtualenv is a bit nicer for setting up virtual environment, since it will provide up-to-date versions of
 # pip/setuptools/wheel which basic `python3 -m venv venv` won't
@@ -39,7 +45,7 @@ if command -v python2 >/dev/null 2>&1; then
 		curl --location --fail https://bootstrap.pypa.io/pip/2.7/get-pip.py | python2
 	fi
 
-	python2 -m pip install --user --upgrade pip setuptools wheel
+	python2 -m pip install --user --upgrade pip 'setuptools<60' wheel
 
 	python2 -m pip install --user 'virtualenv!=20.12.0'
 fi