Numpy 2.0.0 support PRs cherry-pick (#1770)

* [BE] Remove macos x86 build statements, remove python <3.8 (#1745)

* [BE] Remove macos x86 build statements, remove python <3.8

* fix

* deprecate

* Update nightly wheel build numpy to 2.0 (#1747)

* Update wheel build numpy to 2.0 (#1746)

* Update wheel build numpy to 2.0

* aarch64

* Update build_wheel.sh

* Simplify

* Install llvm-openmp for M1 PyTorch builds

Otherwise the binary will be super slow

Also drop `nomkl` as there were never mkl on Apple silicon

* Add numpy 2.0.0rc1 support to conda and wheels (#1768)

* Add numpy 2.0.0rc1 support to conda and wheels

* conda

* conda

* test

* fix

* Add numpy 2.0.0rc1 support to conda and wheels (#1769)

---------

Co-authored-by: Nikita Shulga <[email protected]>

Author: atalman

aarch64_linux/aarch64_ci_setup.sh

@@ -26,11 +26,11 @@ conda create -y -c conda-forge -n "${CONDA_ENV_NAME}" python=${DESIRED_PYTHON}
 conda activate "${CONDA_ENV_NAME}"
 
 if [[ "$DESIRED_PYTHON"  == "3.8" ]]; then
-    NUMPY_VERSION="1.24.4"
+    pip install -q numpy==1.24.4
 else
-    NUMPY_VERSION="1.26.2"
+    pip install -q --pre numpy==2.0.0rc1
 fi
-conda install -y -c conda-forge numpy==${NUMPY_VERSION} pyyaml==6.0.1 patchelf==0.17.2 pygit2==1.13.2 openblas==0.3.25=*openmp* ninja==1.11.1 scons==4.5.2
+conda install -y -c conda-forge pyyaml==6.0.1 patchelf==0.17.2 pygit2==1.13.2 openblas==0.3.25=*openmp* ninja==1.11.1 scons==4.5.2
 
 python --version
 conda --version

conda/build_pytorch.sh

@@ -85,7 +85,7 @@ if [ -z "$ANACONDA_TOKEN" ]; then
 fi
 if [[ -z "$ANACONDA_USER" ]]; then
     # This is the channel that finished packages will be uploaded to
-    ANACONDA_USER=soumith
+    ANACONDA_USER='pytorch'
 fi
 if [[ -z "$GITHUB_ORG" ]]; then
     GITHUB_ORG='pytorch'
@@ -360,12 +360,14 @@ for py_ver in "${DESIRED_PYTHON[@]}"; do
     fi
 
     echo "Calling conda-build at $(date)"
+    # TODO: Remove atalman channel once we can wend numpy from
+    # anaconda or pytorch or pytorch nightly channel
     time CMAKE_ARGS=${CMAKE_ARGS[@]} \
          EXTRA_CAFFE2_CMAKE_FLAGS=${EXTRA_CAFFE2_CMAKE_FLAGS[@]} \
          PYTORCH_GITHUB_ROOT_DIR="$pytorch_rootdir" \
          PYTORCH_BUILD_STRING="$build_string" \
          PYTORCH_MAGMA_CUDA_VERSION="$cuda_nodot" \
-         conda build -c "$ANACONDA_USER" \
+         conda build -c "$ANACONDA_USER" -c atalman \
                      ${NO_TEST:-} \
                      --no-anaconda-upload \
                      --python "$py_ver" \

conda/pytorch-nightly/meta.yaml

@@ -29,10 +29,8 @@ requirements:
     - typing_extensions
     - ninja
     - libuv # [win]
-    - numpy=1.19 # [py <= 39]
-    - numpy=1.21.5 # [py == 310]
-    - numpy=1.23.5 # [py == 311]
-    - numpy=1.26.0 # [py >= 312]
+    - numpy=1.19 # [py == 38]
+    - numpy=2.0.0rc1 # [py >= 39]
     - openssl=1.1.1l # [py >= 38 and py <= 310 and linux]
     - openssl=1.1.1s # [py == 311 and linux]
     - openssl=3.0.12 # [py >= 312 and linux]

manywheel/build_common.sh

@@ -76,15 +76,7 @@ fi
 # in this case
 if [[ -n "$DESIRED_PYTHON" && "$DESIRED_PYTHON" != cp* ]]; then
     python_nodot="$(echo $DESIRED_PYTHON | tr -d m.u)"
-    case ${DESIRED_PYTHON} in
-      3.[6-7]*)
-        DESIRED_PYTHON="cp${python_nodot}-cp${python_nodot}m"
-        ;;
-      # Should catch 3.8+
-      3.*)
-        DESIRED_PYTHON="cp${python_nodot}-cp${python_nodot}"
-        ;;
-    esac
+    DESIRED_PYTHON="cp${python_nodot}-cp${python_nodot}"
 fi
 
 if [[ ${python_nodot} -ge 310 ]]; then
@@ -119,24 +111,15 @@ pushd "$PYTORCH_ROOT"
 python setup.py clean
 retry pip install -qr requirements.txt
 case ${DESIRED_PYTHON} in
-  cp36-cp36m)
-    retry pip install -q numpy==1.11
-    ;;
-  cp3[7-8]*)
+  cp38*)
     retry pip install -q numpy==1.15
     ;;
-  cp310*)
-    retry pip install -q numpy==1.21.2
-    ;;
-  cp311*)
-    retry pip install -q numpy==1.23.1
-    ;;
-  cp312*)
-    retry pip install -q numpy==1.26.1
+  cp31*)
+    retry pip install -q --pre numpy==2.0.0rc1
     ;;
   # Should catch 3.9+
   *)
-    retry pip install -q numpy==1.19.4
+    retry pip install -q --pre numpy==2.0.0rc1
     ;;
 esac
 
@@ -280,7 +263,7 @@ replace_needed_sofiles() {
         patchedname=$3
         if [[ "$origname" != "$patchedname" ]] || [[ "$DESIRED_CUDA" == *"rocm"* ]]; then
             set +e
-            origname=$($PATCHELF_BIN --print-needed $sofile | grep "$origname.*") 
+            origname=$($PATCHELF_BIN --print-needed $sofile | grep "$origname.*")
             ERRCODE=$?
             set -e
             if [ "$ERRCODE" -eq "0" ]; then

manywheel/conda_build.sh

@@ -97,13 +97,8 @@ fi
 whl_tmp_dir="${MAC_PACKAGE_WORK_DIR}/dist"
 mkdir -p "$whl_tmp_dir"
 
-if [[ -n "$CROSS_COMPILE_ARM64" || $(uname -m) == "arm64" ]]; then
-    mac_version='macosx_11_0_arm64'
-    libtorch_arch='arm64'
-else
-    mac_version='macosx_10_9_x86_64'
-    libtorch_arch='x86_64'
-fi
+mac_version='macosx_11_0_arm64'
+libtorch_arch='arm64'
 
 # Create a consistent wheel package name to rename the wheel to
 wheel_filename_new="${TORCH_PACKAGE_NAME}-${build_version}${build_number_prefix}-cp${python_nodot}-none-${mac_version}.whl"
@@ -141,35 +136,31 @@ case $desired_python in
         echo "Using 3.12 deps"
         SETUPTOOLS_PINNED_VERSION=">=68.0.0"
         PYYAML_PINNED_VERSION=">=6.0.1"
-        NUMPY_PINNED_VERSION="==1.26.0"
+        NUMPY_PINNED_VERSION="=2.0.0rc1"
         ;;
     3.11)
         echo "Using 3.11 deps"
         SETUPTOOLS_PINNED_VERSION=">=46.0.0"
         PYYAML_PINNED_VERSION=">=5.3"
-        NUMPY_PINNED_VERSION="==1.23.5"
+        NUMPY_PINNED_VERSION="=2.0.0rc1"
         ;;
     3.10)
         echo "Using 3.10 deps"
         SETUPTOOLS_PINNED_VERSION=">=46.0.0"
         PYYAML_PINNED_VERSION=">=5.3"
-        NUMPY_PINNED_VERSION="=1.21.2"
+        NUMPY_PINNED_VERSION="=2.0.0rc1"
         ;;
     3.9)
         echo "Using 3.9 deps"
         SETUPTOOLS_PINNED_VERSION=">=46.0.0"
         PYYAML_PINNED_VERSION=">=5.3"
-        NUMPY_PINNED_VERSION="=1.19"
+        NUMPY_PINNED_VERSION="=2.0.0rc1"
         ;;
     3.8)
         echo "Using 3.8 deps"
-        if [[ "$(uname -m)" == "arm64" ]]; then
-          SETUPTOOLS_PINNED_VERSION=">=46.0.0"
-          PYYAML_PINNED_VERSION=">=5.3"
-          NUMPY_PINNED_VERSION="=1.19"
-        else
-          NUMPY_PINNED_VERSION="=1.17"
-        fi
+        SETUPTOOLS_PINNED_VERSION=">=46.0.0"
+        PYYAML_PINNED_VERSION=">=5.3"
+        NUMPY_PINNED_VERSION="=1.19"
         ;;
     *)
         echo "Using default deps"
@@ -182,30 +173,24 @@ tmp_env_name="wheel_py$python_nodot"
 conda create ${EXTRA_CONDA_INSTALL_FLAGS} -yn "$tmp_env_name" python="$desired_python"
 source activate "$tmp_env_name"
 
-retry conda install ${EXTRA_CONDA_INSTALL_FLAGS} -yq "numpy${NUMPY_PINNED_VERSION}" nomkl "setuptools${SETUPTOOLS_PINNED_VERSION}" "pyyaml${PYYAML_PINNED_VERSION}" typing_extensions requests
-
-if [[ "$(uname -m)" == "arm64" ]]; then
-  retry conda install ${EXTRA_CONDA_INSTALL_FLAGS} -yq cmake ninja
+if [[ $desired_python != "3.8" ]]; then
+    pip install -q --pre numpy=${NUMPY_PINNED_VERSION}
 else
-  retry conda install ${EXTRA_CONDA_INSTALL_FLAGS} -yq cmake ninja mkl-include==2022.2.1 mkl-static==2022.2.1 -c intel
+    retry conda install ${EXTRA_CONDA_INSTALL_FLAGS}  -yq "numpy${NUMPY_PINNED_VERSION}"
 fi
+retry conda install ${EXTRA_CONDA_INSTALL_FLAGS} -yq  llvm-openmp=14.0.6 cmake ninja "setuptools${SETUPTOOLS_PINNED_VERSION}" "pyyaml${PYYAML_PINNED_VERSION}" typing_extensions requests
 retry pip install -qr "${pytorch_rootdir}/requirements.txt" || true
 
 # For USE_DISTRIBUTED=1 on macOS, need libuv and pkg-config to find libuv.
 export USE_DISTRIBUTED=1
 retry conda install ${EXTRA_CONDA_INSTALL_FLAGS} -yq libuv pkg-config
 
-if [[ -n "$CROSS_COMPILE_ARM64" || "$(uname -m)" == "arm64" ]]; then
-    if [[ -n "$CROSS_COMPILE_ARM64" ]]; then
-        export CMAKE_OSX_ARCHITECTURES=arm64
-    fi
-    export USE_MKLDNN=OFF
-    export USE_QNNPACK=OFF
-    export BUILD_TEST=OFF
-elif [[ "$(uname -m)" == "x86_64" ]]; then
-    retry conda install ${EXTRA_CONDA_INSTALL_FLAGS} -yq llvmdev=9
-    export USE_LLVM="${CONDA_PREFIX}"
+if [[ -n "$CROSS_COMPILE_ARM64" ]]; then
+    export CMAKE_OSX_ARCHITECTURES=arm64
 fi
+export USE_MKLDNN=OFF
+export USE_QNNPACK=OFF
+export BUILD_TEST=OFF
 
 pushd "$pytorch_rootdir"
 echo "Calling setup.py bdist_wheel at $(date)"
@@ -289,9 +274,4 @@ else
     zip -rq "$PYTORCH_FINAL_PACKAGE_DIR/libtorch-macos-${libtorch_arch}-$PYTORCH_BUILD_VERSION.zip" libtorch
     cp "$PYTORCH_FINAL_PACKAGE_DIR/libtorch-macos-${libtorch_arch}-$PYTORCH_BUILD_VERSION.zip"  \
        "$PYTORCH_FINAL_PACKAGE_DIR/libtorch-macos-${libtorch_arch}-latest.zip"
-    if [[ "${libtorch_arch}" == "x86_64" ]]; then
-      # For backward compatibility make unarched latest to point to x86_64
-      cp "$PYTORCH_FINAL_PACKAGE_DIR/libtorch-macos-${libtorch_arch}-$PYTORCH_BUILD_VERSION.zip"  \
-         "$PYTORCH_FINAL_PACKAGE_DIR/libtorch-macos-latest.zip"
-    fi
 fi

wheel/build_wheel.sh

@@ -9,13 +9,11 @@ FOR %%v IN (%DESIRED_PYTHON%) DO (
     set PYTHON_VERSION_STR=%%v
     set PYTHON_VERSION_STR=!PYTHON_VERSION_STR:.=!
     conda remove -n py!PYTHON_VERSION_STR! --all -y || rmdir %CONDA_HOME%\envs\py!PYTHON_VERSION_STR! /s
-    if "%%v" == "3.7" call conda create -n py!PYTHON_VERSION_STR! -y -q numpy=1.11 "mkl=2020.2" pyyaml boto3 cmake ninja typing_extensions python=%%v
     if "%%v" == "3.8" call conda create -n py!PYTHON_VERSION_STR! -y -q numpy=1.11 "mkl=2020.2" pyyaml boto3 cmake ninja typing_extensions python=%%v
     if "%%v" == "3.9" call conda create -n py!PYTHON_VERSION_STR! -y -q numpy>=1.11 "mkl=2020.2" pyyaml boto3 cmake ninja typing_extensions python=%%v
     if "%%v" == "3.10" call conda create -n py!PYTHON_VERSION_STR! -y -q -c=conda-forge numpy=1.21.3 "mkl=2020.2" intel-openmp=2023.2.0 pyyaml boto3 "cmake=3.19.6" ninja typing_extensions python=%%v
     if "%%v" == "3.11" call conda create -n py!PYTHON_VERSION_STR! -y -q -c=conda-forge numpy=1.23.4 "mkl=2020.2" intel-openmp=2023.2.0 pyyaml boto3 "cmake=3.19.6" ninja typing_extensions python=%%v
     if "%%v" == "3.12" call conda create -n py!PYTHON_VERSION_STR! -y -q -c=conda-forge numpy=1.26.0 "mkl=2023.1" intel-openmp=2023.2.0 pyyaml boto3 "cmake=3.19.6" ninja typing_extensions python=%%v
-    if "%%v" == "3" call conda create -n py!PYTHON_VERSION_STR! -y -q numpy=1.11 "mkl=2020.2" pyyaml boto3 cmake ninja typing_extensions python=%%v
 )
 endlocal
 

windows/condaenv.bat

@@ -35,8 +35,6 @@ if "%DESIRED_PYTHON%" == "3.11" set "PYTHON_INSTALLER_URL=https://www.python.org
 if "%DESIRED_PYTHON%" == "3.10" set "PYTHON_INSTALLER_URL=https://www.python.org/ftp/python/3.10.0/python-3.10.0-amd64.exe"
 if "%DESIRED_PYTHON%" == "3.9" set "PYTHON_INSTALLER_URL=https://www.python.org/ftp/python/3.9.0/python-3.9.0-amd64.exe"
 if "%DESIRED_PYTHON%" == "3.8" set "PYTHON_INSTALLER_URL=https://www.python.org/ftp/python/3.8.2/python-3.8.2-amd64.exe"
-if "%DESIRED_PYTHON%" == "3.7" set "PYTHON_INSTALLER_URL=https://www.python.org/ftp/python/3.7.9/python-3.7.9-amd64.exe"
-if "%DESIRED_PYTHON%" == "3.6" set "PYTHON_INSTALLER_URL=https://www.python.org/ftp/python/3.6.8/python-3.6.8-amd64.exe"
 if "%PYTHON_INSTALLER_URL%" == "" (
     echo Python %DESIRED_PYTHON% not supported yet
 )
@@ -54,7 +52,13 @@ if errorlevel 1 exit /b 1
 
 set "PATH=%CD%\Python%PYTHON_VERSION%\Scripts;%CD%\Python;%PATH%"
 
-pip install -q numpy protobuf
+
+if "%DESIRED_PYTHON%" == "3.12" pip install -q --pre numpy==2.0.0rc1 protobuf
+if "%DESIRED_PYTHON%" == "3.11" pip install -q --pre numpy==2.0.0rc1 protobuf
+if "%DESIRED_PYTHON%" == "3.10" pip install -q --pre numpy==2.0.0rc1 protobuf
+if "%DESIRED_PYTHON%" == "3.9" pip install -q --pre numpy==2.0.0rc1 protobuf
+if "%DESIRED_PYTHON%" == "3.8" pip install -q numpy protobuf
+
 if errorlevel 1 exit /b 1
 
 for /F "delims=" %%i in ('where /R "%PYTORCH_FINAL_PACKAGE_DIR:/=\%" *.whl') do pip install "%%i"