From cfc3b570a6a682f4c9dfc70827152e8b67d0a2d2 Mon Sep 17 00:00:00 2001 From: Nikita Shulga Date: Wed, 16 Feb 2022 07:39:16 -0800 Subject: [PATCH 1/2] Delete stale packaging scripts `osx_wheel.sh` and `linux_manywheel.sh` has not been functional for a while, and are not referenced from any of the CI scripts Updated README.md to reflect how release builds are currently made --- packaging/README.md | 89 +----------------------------- packaging/wheel/linux_manywheel.sh | 59 -------------------- packaging/wheel/osx_wheel.sh | 50 ----------------- 3 files changed, 2 insertions(+), 196 deletions(-) delete mode 100644 packaging/wheel/linux_manywheel.sh delete mode 100644 packaging/wheel/osx_wheel.sh diff --git a/packaging/README.md b/packaging/README.md index 7d3c5f7831b..fde6e66e0ca 100644 --- a/packaging/README.md +++ b/packaging/README.md @@ -1,90 +1,5 @@ # Building torchvision packages for release -## Anaconda packages +TorchVision release packages are build by invoking `build_wheel.sh` and `build_conda.sh` for all OSes, compute and python version permutation -### Linux - -```bash -nvidia-docker run -it --ipc=host --rm -v $(pwd):/remote soumith/conda-cuda bash -pushd remote/conda - -./build_vision.sh 9.0 -./build_vision.sh 10.0 -./build_vision.sh cpu - -# copy packages over to /remote -# exit docker -# anaconda upload -u pytorch torchvision*.bz2 -``` - -### OSX - -```bash -# create a fresh anaconda environment / install and activate it -conda install -y conda-build anaconda-client -./build_vision.sh cpu - -# copy packages over to /remote -# exit docker -# anaconda upload -u pytorch torchvision*.bz2 -``` - -### Windows - -```bash -# Open `Git Bash` and change dir to `conda` -./build_vision.sh 9.0 -./build_vision.sh 10.0 -./build_vision.sh cpu - -# copy packages to a output directory -# anaconda upload -u pytorch torchvision*.bz2 -``` - -## Wheels - -### Linux - -pushd wheel - -```bash -nvidia-docker run -it --ipc=host --rm -v $(pwd):/remote soumith/manylinux-cuda90:latest bash -cd remote -./linux_manywheel.sh cu90 - -rm -rf /usr/local/cuda* -./linux_manywheel.sh cpu -``` - -```bash -nvidia-docker run -it --ipc=host --rm -v $(pwd):/remote soumith/manylinux-cuda100:latest bash -cd remote -./linux_manywheel.sh cu100 -``` - -wheels are in the folders `cpu`, `cu90`, `cu100`. - -You can upload the `cu90` wheels to twine with `twine upload *.whl`. -Which wheels we upload depends on which wheels PyTorch uploads as default, and right now, it's `cu90`. - -### OSX - -```bash -pushd wheel -./osx_wheel.sh -``` - -### Windows - -```cmd -set PYTORCH_REPO=pytorch - -pushd windows -call build_vision.bat 90 0.3.0 1 -call build_vision.bat 100 0.3.0 1 -call build_vision.bat cpu 0.3.0 1 -``` - -wheels are in the current folder. - -You can upload them to twine with `twine upload *.whl` +OS/Python/CUDA matrix is defined in https://github.com/pytorch/vision/blob/main/.circleci/regenerate.py diff --git a/packaging/wheel/linux_manywheel.sh b/packaging/wheel/linux_manywheel.sh deleted file mode 100644 index 965c8150748..00000000000 --- a/packaging/wheel/linux_manywheel.sh +++ /dev/null @@ -1,59 +0,0 @@ -#!/bin/bash -set -ex - -if [ "$#" -ne 1 ]; then - echo "Illegal number of parameters. Pass cuda version" - echo "CUDA version should be cu92, cu100 or cpu" - exit 1 -fi -export CUVER="$1" # cu[0-9]* cpu - -if [[ "$CUVER" == "cu102" ]]; then - cu_suffix="" -else - cu_suffix="+$CUVER" -fi - -export OUT_DIR="/remote/$CUVER" - -pushd /opt/python -DESIRED_PYTHON=(*/) -popd -for desired_py in "${DESIRED_PYTHON[@]}"; do - python_installations+=("/opt/python/$desired_py") -done - -OLD_PATH=$PATH -cd /tmp -rm -rf vision -git clone https://github.com/pytorch/vision - -cd /tmp/vision - -for PYDIR in "${python_installations[@]}"; do - export PATH=$PYDIR/bin:$OLD_PATH - pip install --upgrade pip - pip install numpy pyyaml future - - pip uninstall -y torch || true - pip uninstall -y torch_nightly || true - - export TORCHVISION_PYTORCH_DEPENDENCY_NAME=torch_nightly - pip install torch_nightly -f https://download.pytorch.org/whl/nightly/$CUVER/torch_nightly.html - # CPU/CUDA variants of PyTorch have ABI compatible PyTorch for - # the CPU only bits. Therefore, we - # strip off the local package qualifier, but ONLY if we're - # doing a CPU build. - if [[ "$CUVER" == "cpu" ]]; then - export TORCHVISION_PYTORCH_DEPENDENCY_VERSION="$(pip show torch_nightly | grep ^Version: | sed 's/Version: \+//' | sed 's/+.\+//')" - else - export TORCHVISION_PYTORCH_DEPENDENCY_VERSION="$(pip show torch_nightly | grep ^Version: | sed 's/Version: \+//')" - fi - echo "Building against ${TORCHVISION_PYTORCH_DEPENDENCY_VERSION}" - - pip install ninja - python setup.py clean - python setup.py bdist_wheel - mkdir -p $OUT_DIR - cp dist/*.whl $OUT_DIR/ -done diff --git a/packaging/wheel/osx_wheel.sh b/packaging/wheel/osx_wheel.sh deleted file mode 100644 index 035a6522fda..00000000000 --- a/packaging/wheel/osx_wheel.sh +++ /dev/null @@ -1,50 +0,0 @@ -if [[ ":$PATH:" == *"conda"* ]]; then - echo "existing anaconda install in PATH, remove it and run script" - exit 1 -fi -# download and activate anaconda -rm -rf ~/minconda_wheel_env_tmp -wget -q https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh && \ - chmod +x Miniconda3-latest-MacOSX-x86_64.sh && \ - ./Miniconda3-latest-MacOSX-x86_64.sh -b -p ~/minconda_wheel_env_tmp && \ - rm Miniconda3-latest-MacOSX-x86_64.sh - -. ~/minconda_wheel_env_tmp/bin/activate - - -export OUT_DIR=~/torchvision_wheels - -export MACOSX_DEPLOYMENT_TARGET=10.9 CC=clang CXX=clang++ - -pushd /tmp -rm -rf vision -git clone https://github.com/pytorch/vision -pushd vision - -desired_pythons=( "3.7" "3.8" "3.9" ) -# for each python -for desired_python in "${desired_pythons[@]}" -do - # create and activate python env - env_name="env$desired_python" - conda create -yn $env_name python="$desired_python" - conda activate $env_name - - pip uninstall -y torch || true - pip uninstall -y torch_nightly || true - - export TORCHVISION_PYTORCH_DEPENDENCY_NAME=torch_nightly - pip install torch_nightly -f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html - export TORCHVISION_PYTORCH_DEPENDENCY_VERSION="$(pip show torch_nightly | grep ^Version: | sed 's/Version: *//')" - echo "Building against ${TORCHAUDIO_PYTORCH_DEPENDENCY_VERSION}" - - # install torchvision dependencies - pip install ninja scipy pytest - - python setup.py clean - python setup.py bdist_wheel - mkdir -p $OUT_DIR - cp dist/*.whl $OUT_DIR/ -done -popd -popd From b0496a678332121af0374390c540bfde8f09c498 Mon Sep 17 00:00:00 2001 From: Nikita Shulga Date: Wed, 16 Feb 2022 09:09:50 -0800 Subject: [PATCH 2/2] Update README.md --- packaging/README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/packaging/README.md b/packaging/README.md index fde6e66e0ca..3ceac53030e 100644 --- a/packaging/README.md +++ b/packaging/README.md @@ -1,5 +1,6 @@ # Building torchvision packages for release -TorchVision release packages are build by invoking `build_wheel.sh` and `build_conda.sh` for all OSes, compute and python version permutation +TorchVision release packages are built by using `build_wheel.sh` and `build_conda.sh` for all permutations of +supported operating systems, compute platforms and python versions. -OS/Python/CUDA matrix is defined in https://github.com/pytorch/vision/blob/main/.circleci/regenerate.py +OS/Python/Compute matrix is defined in https://github.com/pytorch/vision/blob/main/.circleci/regenerate.py