Fix Release Pipeline for distutils (#598)

Author: waahm7

.github/workflows/ci.yml

@@ -26,7 +26,6 @@ jobs:
           - x64
           - x86
         python:
-          - cp37-cp37m
           - cp38-cp38
           - cp39-cp39
     permissions:
@@ -52,7 +51,6 @@ jobs:
           - x86
           - aarch64
         python:
-          - cp37-cp37m
           - cp38-cp38
           - cp39-cp39
           - cp310-cp310
@@ -85,7 +83,6 @@ jobs:
           - x64
           - aarch64
         python:
-          - cp37-cp37m
           - cp38-cp38
           - cp39-cp39
           - cp310-cp310

codebuild/cd/manylinux-x64-build.yml

@@ -12,8 +12,6 @@ phases:
   build:
     commands:
       - echo Build started on `date`
-      - /opt/python/cp37-cp37m/bin/python setup.py sdist bdist_wheel
-      - auditwheel repair --plat manylinux1_x86_64 dist/awscrt-*cp37-cp37m-linux_x86_64.whl
       - /opt/python/cp38-cp38/bin/python setup.py sdist bdist_wheel
       - auditwheel repair --plat manylinux1_x86_64 dist/awscrt-*cp38-cp38-linux_x86_64.whl
       - /opt/python/cp39-cp39/bin/python setup.py sdist bdist_wheel

codebuild/cd/manylinux-x86-build.yml

@@ -12,8 +12,6 @@ phases:
   build:
     commands:
       - echo Build started on `date`
-      - /opt/python/cp37-cp37m/bin/python setup.py sdist bdist_wheel
-      - auditwheel repair --plat manylinux1_i686 dist/awscrt-*cp37-cp37m-linux_i686.whl
       - /opt/python/cp38-cp38/bin/python setup.py sdist bdist_wheel
       - auditwheel repair --plat manylinux1_i686 dist/awscrt-*cp38-cp38-linux_i686.whl
       - /opt/python/cp39-cp39/bin/python setup.py sdist bdist_wheel

continuous-delivery/build-wheels-manylinux2014-aarch64.sh

@@ -4,9 +4,6 @@ set -ex
 
 /opt/python/cp39-cp39/bin/python ./continuous-delivery/update-version.py
 
-/opt/python/cp37-cp37m/bin/python setup.py sdist bdist_wheel
-auditwheel repair --plat manylinux2014_aarch64 dist/awscrt-*cp37*.whl
-
 /opt/python/cp38-cp38/bin/python setup.py sdist bdist_wheel
 auditwheel repair --plat manylinux2014_aarch64 dist/awscrt-*cp38*.whl
 
@@ -23,6 +20,10 @@ auditwheel repair --plat manylinux2014_aarch64 dist/awscrt-*cp311*.whl
 # The 3.11 wheel uses the stable ABI, so it works with newer versions too.
 
 # We are using the Python 3.13 stable ABI from Python 3.13 onwards because of deprecated functions.
+# Manylinux images don't contain setuptools from Python 3.13, so we need to install it.
+# Install in a custom location due to access issues.
+/opt/python/cp313-cp313/bin/python -m pip install --target ./local -r requirements-dev.txt
+export PYTHONPATH=./local:$PYTHONPATH
 /opt/python/cp313-cp313/bin/python setup.py sdist bdist_wheel
 auditwheel repair --plat manylinux2014_aarch64 dist/awscrt-*cp313*.whl
 

continuous-delivery/build-wheels-manylinux2014-x86_64.sh

@@ -4,9 +4,6 @@ set -ex
 
 /opt/python/cp39-cp39/bin/python ./continuous-delivery/update-version.py
 
-/opt/python/cp37-cp37m/bin/python setup.py sdist bdist_wheel
-auditwheel repair --plat manylinux2014_x86_64 dist/awscrt-*cp37*.whl
-
 /opt/python/cp38-cp38/bin/python setup.py sdist bdist_wheel
 auditwheel repair --plat manylinux2014_x86_64 dist/awscrt-*cp38*.whl
 
@@ -23,6 +20,10 @@ auditwheel repair --plat manylinux2014_x86_64 dist/awscrt-*cp311*.whl
 # The 3.11 wheel uses the stable ABI, so it works with newer versions too.
 
 # We are using the Python 3.13 stable ABI from Python 3.13 onwards because of deprecated functions.
+# Manylinux images don't contain setuptools from Python 3.13, so we need to install it.
+# Install in a custom location due to access issues.
+/opt/python/cp313-cp313/bin/python -m pip install --target ./local -r requirements-dev.txt
+export PYTHONPATH=./local:$PYTHONPATH
 /opt/python/cp313-cp313/bin/python setup.py sdist bdist_wheel
 auditwheel repair --plat manylinux2014_x86_64 dist/awscrt-*cp313*.whl
 

continuous-delivery/build-wheels-musllinux-1-1-aarch64.sh

@@ -4,9 +4,6 @@ set -ex
 
 /opt/python/cp39-cp39/bin/python ./continuous-delivery/update-version.py
 
-/opt/python/cp37-cp37m/bin/python setup.py sdist bdist_wheel
-auditwheel repair --plat musllinux_1_1_aarch64 dist/awscrt-*cp37*.whl
-
 /opt/python/cp38-cp38/bin/python setup.py sdist bdist_wheel
 auditwheel repair --plat musllinux_1_1_aarch64 dist/awscrt-*cp38*.whl
 
@@ -23,6 +20,10 @@ auditwheel repair --plat musllinux_1_1_aarch64 dist/awscrt-*cp311*.whl
 # The 3.11 wheel uses the stable ABI, so it works with newer versions too.
 
 # We are using the Python 3.13 stable ABI from Python 3.13 onwards because of deprecated functions.
+# Manylinux images don't contain setuptools from Python 3.13, so we need to install it.
+# Install in a custom location due to access issues.
+/opt/python/cp313-cp313/bin/python -m pip install --target ./local -r requirements-dev.txt
+export PYTHONPATH=./local:$PYTHONPATH
 /opt/python/cp313-cp313/bin/python setup.py sdist bdist_wheel
 auditwheel repair --plat musllinux_1_1_aarch64 dist/awscrt-*cp313*.whl
 

continuous-delivery/build-wheels-musllinux-1-1-x86_64.sh

@@ -4,9 +4,6 @@ set -ex
 
 /opt/python/cp39-cp39/bin/python ./continuous-delivery/update-version.py
 
-/opt/python/cp37-cp37m/bin/python setup.py sdist bdist_wheel
-auditwheel repair --plat musllinux_1_1_x86_64 dist/awscrt-*cp37*.whl
-
 /opt/python/cp38-cp38/bin/python setup.py sdist bdist_wheel
 auditwheel repair --plat musllinux_1_1_x86_64 dist/awscrt-*cp38*.whl
 
@@ -23,6 +20,10 @@ auditwheel repair --plat musllinux_1_1_x86_64 dist/awscrt-*cp311*.whl
 # The 3.11 wheel uses the stable ABI, so it works with newer versions too.
 
 # We are using the Python 3.13 stable ABI from Python 3.13 onwards because of deprecated functions.
+# Manylinux images don't contain setuptools from Python 3.13, so we need to install it.
+# Install in a custom location due to access issues.
+/opt/python/cp313-cp313/bin/python -m pip install --target ./local -r requirements-dev.txt
+export PYTHONPATH=./local:$PYTHONPATH
 /opt/python/cp313-cp313/bin/python setup.py sdist bdist_wheel
 auditwheel repair --plat musllinux_1_1_x86_64 dist/awscrt-*cp313*.whl
 

continuous-delivery/build-wheels-osx.sh

@@ -5,7 +5,6 @@ set -ex
 
 /Library/Frameworks/Python.framework/Versions/3.9/bin/python3 ./continuous-delivery/update-version.py
 
-/Library/Frameworks/Python.framework/Versions/3.7/bin/python3 setup.py sdist bdist_wheel
 /Library/Frameworks/Python.framework/Versions/3.8/bin/python3 setup.py sdist bdist_wheel
 /Library/Frameworks/Python.framework/Versions/3.9/bin/python3 setup.py sdist bdist_wheel
 /Library/Frameworks/Python.framework/Versions/3.10/bin/python3 setup.py sdist bdist_wheel

continuous-delivery/build-wheels-win32.bat

@@ -1,7 +1,6 @@
 
 "C:\Program Files (x86)\Python39-32\python.exe" .\continuous-delivery\update-version.py || goto error
 
-"C:\Program Files (x86)\Python37-32\python.exe" setup.py sdist bdist_wheel || goto error
 "C:\Program Files (x86)\Python38-32\python.exe" setup.py sdist bdist_wheel || goto error
 "C:\Program Files (x86)\Python39-32\python.exe" setup.py sdist bdist_wheel || goto error
 "C:\Program Files (x86)\Python310-32\python.exe" setup.py sdist bdist_wheel || goto error

continuous-delivery/build-wheels-win64.bat

@@ -1,6 +1,5 @@
 "C:\Program Files\Python39\python.exe" continuous-delivery\update-version.py || goto error
 
-"C:\Program Files\Python37\python.exe" setup.py sdist bdist_wheel || goto error
 "C:\Program Files\Python38\python.exe" setup.py sdist bdist_wheel || goto error
 "C:\Program Files\Python39\python.exe" setup.py sdist bdist_wheel || goto error
 "C:\Program Files\Python310\python.exe" setup.py sdist bdist_wheel || goto error

setup.py

@@ -1,8 +1,6 @@
 # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
 # SPDX-License-Identifier: Apache-2.0.
-
 import codecs
-import distutils.ccompiler
 import glob
 import os
 import os.path
@@ -78,6 +76,7 @@ def determine_generator_args():
     if sys.platform == 'win32':
         try:
             # See which compiler python picks
+            import distutils.ccompiler
             compiler = distutils.ccompiler.new_compiler()
             compiler.initialize()
 
@@ -305,8 +304,12 @@ def run(self):
 class bdist_wheel_abi3(bdist_wheel):
     def get_tag(self):
         python, abi, plat = super().get_tag()
-        if python.startswith("cp") and sys.version_info >= (3, 11):
-            # on CPython, our wheels are abi3 and compatible back to 3.11
+        # on CPython, our wheels are abi3 and compatible back to 3.11
+        if python.startswith("cp") and sys.version_info >= (3, 13):
+            # 3.13 deprecates PyWeakref_GetObject(), adds alternative
+            return "cp313", "abi3", plat
+        elif python.startswith("cp") and sys.version_info >= (3, 11):
+            # 3.11 is the first stable ABI that has everything we need
             return "cp311", "abi3", plat
 
         return python, abi, plat
@@ -326,6 +329,11 @@ def awscrt_ext():
     libraries.reverse()
 
     if sys.platform == 'win32':
+        # distutils is deprecated in Python 3.10 and removed in 3.12. However, it still works because Python defines a compatibility interface as long as setuptools is installed.
+        # We don't have an official alternative for distutils.ccompiler as of September 2024. See: https://github.com/pypa/setuptools/issues/2806
+        # Once that issue is resolved, we can migrate to the official solution.
+        # For now, restrict distutils to Windows only, where it's needed.
+        import distutils.ccompiler
         # the windows apis being used under the hood. Since we're static linking we have to follow the entire chain down
         libraries += ['Secur32', 'Crypt32', 'Advapi32', 'NCrypt', 'BCrypt', 'Kernel32', 'Ws2_32', 'Shlwapi']
         # Ensure that debug info is in the obj files, and that it is linked into the .pyd so that
@@ -374,7 +382,7 @@ def awscrt_ext():
         # rare cases where that didn't happen, so let's be explicit.
         extra_link_args += ['-pthread']
 
-    if distutils.ccompiler.get_default_compiler() != 'msvc':
+    if sys.platform != 'win32' or distutils.ccompiler.get_default_compiler() != 'msvc':
         extra_compile_args += ['-Wno-strict-aliasing', '-std=gnu99']
 
         # treat warnings as errors in development mode