Make Pip willing to hide system Python modules

Makefile

@@ -65,7 +65,7 @@ the appliance images to, for example:
 You might also want to build just for one architecture and load into your
 Docker daemon. We have a 'load_docker' target for this.
 
-    make load_docker arch=amd64
+    make load_docker arch=linux/amd64
 
 If Docker is not installed, Docker-related targets tasks and tests will be skipped. The
 same can be achieved by setting TOIL_DOCKER_REGISTRY to an empty string.

docker/Dockerfile.py

@@ -182,18 +182,22 @@ def heredoc(s):
     # The stock pip is too old and can't install from sdist with extras
     RUN curl -sS https://bootstrap.pypa.io/get-pip.py | {python}
 
-    # Include virtualenv, as it is still the recommended way to deploy pipelines
-    RUN {pip} install --upgrade virtualenv==20.25.1
-
-    # Install s3am (--never-download prevents silent upgrades to pip, wheel and setuptools)
-    # Install setuptools within the virtual environment to properly access distutils due to PEP 632 and gh-95299 in Python 3.12 release notes
-    # https://docs.python.org/3/whatsnew/3.12.html#summary-release-highlights
-    RUN virtualenv --python {python} --never-download /home/s3am \
-        && /home/s3am/bin/pip install setuptools \
-        && /home/s3am/bin/pip install s3am==2.0 \
-        && ln -s /home/s3am/bin/s3am /usr/local/bin/
-
-    RUN {pip} install --upgrade setuptools==69.2.0
+    # Include virtualenv, as it is still the recommended way to deploy
+    # pipelines.
+    #
+    # We need to --ignore-installed here to allow shadowing system packages
+    # from apt in /usr/lib/python3/dist-packages when the installed package
+    # needs newer versions. We just hope that doesn't break the Ubuntu system
+    # too badly when we're actually on the system Python, or if Toil needs to
+    # upgrade a distutils or setuptools dependency. On the deadsnakes Pythons,
+    # installations into the version-specific package directory won't be seen
+    # by the system Python which is a different version.
+    #
+    # TODO: Change to nested virtual environments and .pth files and teach Toil
+    # to just ship the user-level one for hot deploy.
+    RUN {pip} install --ignore-installed --upgrade 'virtualenv>=20.25.1,<21'
+
+    RUN {pip} install --ignore-installed --upgrade 'setuptools>=80,<81'
 
     # Fix for https://issues.apache.org/jira/browse/MESOS-3793
     ENV MESOS_LAUNCHER=posix
@@ -222,7 +226,7 @@ def heredoc(s):
 
     # This component changes most frequently and keeping it last maximizes Docker cache hits.
     COPY {sdistName} .
-    RUN {pip} install {sdistName}[all]
+    RUN {pip} install --ignore-installed --upgrade {sdistName}[all]
     RUN rm {sdistName}
 
     # We intentionally inherit the default ENTRYPOINT and CMD from the base image, to the effect

src/toil/test/provisioners/aws/awsProvisionerTest.py

@@ -526,7 +526,7 @@ def _runScript(self, toilOptions):
         runCommand = [
             self.python(),
             self.script(),
-            "--fileToSort=/home/s3am/bin/asadmin",
+            "--fileToSort=/etc/passwd",
             "--sortMemory=0.6G",
             "--mergeMemory=3.0G",
         ]

src/toil/test/provisioners/gceProvisionerTest.py

@@ -341,7 +341,7 @@ def _runScript(self, toilOptions):
         runCommand = [
             "/home/venv/bin/python",
             "/home/sort.py",
-            "--fileToSort=/home/s3am/bin/asadmin",
+            "--fileToSort=/etc/passwd",
             "--sortMemory=0.6G",
             "--mergeMemory=3.0G",
         ]