Skip to content

Commit c0485a7

Browse files
datumboxdatumbox
committed
[v0.8.X] .circleci: Add Python 3.9 to CI (pytorch#3063)
# Conflicts: # .circleci/config.yml # .circleci/config.yml.in # .circleci/unittest/linux/scripts/environment.yml # .circleci/unittest/windows/scripts/environment.yml # packaging/torchvision/meta.yaml
1 parent 7bf6e7b commit c0485a7

File tree

10 files changed

+911
-123
lines changed

10 files changed

+911
-123
lines changed

‎.circleci/config.yml

Lines changed: 849 additions & 105 deletions
Large diffs are not rendered by default.

‎.circleci/config.yml.in

Lines changed: 22 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -401,7 +401,11 @@ jobs:
401401
set -x
402402
eval "$('/C/tools/miniconda3/Scripts/conda.exe' 'shell.bash' 'hook')"
403403
conda env remove -n python${PYTHON_VERSION} || true
404-
conda create -yn python${PYTHON_VERSION} python=${PYTHON_VERSION}
404+
CONDA_CHANNEL_FLAGS=""
405+
if [[ "${PYTHON_VERSION}" = 3.9 ]]; then
406+
CONDA_CHANNEL_FLAGS="-c=conda-forge"
407+
fi
408+
conda create ${CONDA_CHANNEL_FLAGS} -yn python${PYTHON_VERSION} python=${PYTHON_VERSION}
405409
conda activate python${PYTHON_VERSION}
406410
conda install Pillow
407411
conda install -v -y -c pytorch-nightly pytorch
@@ -426,7 +430,11 @@ jobs:
426430
command: |
427431
set -x
428432
eval "$('/C/tools/miniconda3/Scripts/conda.exe' 'shell.bash' 'hook')"
429-
conda env remove -n python${PYTHON_VERSION} || true
433+
CONDA_CHANNEL_FLAGS=""
434+
if [[ "${PYTHON_VERSION}" = 3.9 ]]; then
435+
CONDA_CHANNEL_FLAGS="-c=conda-forge"
436+
fi
437+
conda create ${CONDA_CHANNEL_FLAGS} -yn python${PYTHON_VERSION} python=${PYTHON_VERSION}
430438
conda create -yn python${PYTHON_VERSION} python=${PYTHON_VERSION}
431439
conda activate python${PYTHON_VERSION}
432440
pip install $(ls ~/workspace/torchvision*.whl) --pre -f https://download.pytorch.org/whl/nightly/torch_nightly.html
@@ -491,9 +499,14 @@ jobs:
491499
name: Generate cache key
492500
# This will refresh cache on Sundays, nightly build should generate new cache.
493501
command: echo "$(date +"%Y-%U")" > .circleci-weekly
502+
- restore_cache:
503+
{% raw %}
504+
keys:
505+
- env-v2-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }}
506+
{% endraw %}
494507
- run:
495508
name: Setup
496-
command: docker run -e PYTHON_VERSION -t --gpus all -v $PWD:$PWD -w $PWD "${image_name}" .circleci/unittest/linux/scripts/setup_env.sh
509+
command: docker run -t --gpus all -v $PWD:$PWD -w $PWD -e PYTHON_VERSION "${image_name}" .circleci/unittest/linux/scripts/setup_env.sh
497510
- save_cache:
498511
{% raw %}
499512
key: env-v2-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }}
@@ -503,7 +516,7 @@ jobs:
503516
- env
504517
- run:
505518
name: Install torchvision
506-
command: docker run -t --gpus all -v $PWD:$PWD -w $PWD -e UPLOAD_CHANNEL "${image_name}" .circleci/unittest/linux/scripts/install.sh
519+
command: docker run -t --gpus all -v $PWD:$PWD -w $PWD -e UPLOAD_CHANNEL -e CU_VERSION "${image_name}" .circleci/unittest/linux/scripts/install.sh
507520
- run:
508521
name: Run tests
509522
command: docker run -t --gpus all -v $PWD:$PWD -w $PWD "${image_name}" .circleci/unittest/linux/scripts/run_test.sh
@@ -565,6 +578,11 @@ jobs:
565578
name: Generate cache key
566579
# This will refresh cache on Sundays, nightly build should generate new cache.
567580
command: echo "$(date +"%Y-%U")" > .circleci-weekly
581+
- restore_cache:
582+
{% raw %}
583+
keys:
584+
- env-v1-windows-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/windows/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }}
585+
{% endraw %}
568586
- run:
569587
name: Setup
570588
command: .circleci/unittest/windows/scripts/setup_env.sh

‎.circleci/regenerate.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@
1919
import os.path
2020

2121

22-
PYTHON_VERSIONS = ["3.6", "3.7", "3.8"]
22+
PYTHON_VERSIONS = ["3.6", "3.7", "3.8", "3.9"]
2323

2424

2525
def build_workflows(prefix='', filter_branch=None, upload=False, indentation=6, windows_latest_only=False):

‎.circleci/unittest/linux/scripts/environment.yml

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,18 +1,20 @@
11
channels:
22
- pytorch
33
- defaults
4+
# using conda-forge for python v3.9+
5+
- conda-forge
46
dependencies:
57
- numpy
68
- pytest
79
- pytest-cov
810
- codecov
911
- pip
1012
- libpng
11-
- jpeg
13+
- jpeg <= 9b
1214
- ffmpeg=4.2
1315
- ca-certificates
16+
- av
1417
- pip:
1518
- future
1619
- pillow>=4.1.1
1720
- scipy
18-
- av

‎.circleci/unittest/linux/scripts/install.sh

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,8 +22,14 @@ else
2222
version="$(python -c "print('.'.join(\"${CUDA_VERSION}\".split('.')[:2]))")"
2323
cudatoolkit="cudatoolkit=${version}"
2424
fi
25+
26+
CONDA_CHANNEL_FLAGS=""
27+
if [[ "$(python --version)" = *3.9* ]]; then
28+
CONDA_CHANNEL_FLAGS="-c=conda-forge"
29+
fi
30+
2531
printf "Installing PyTorch with %s\n" "${cudatoolkit}"
26-
conda install -y -c "pytorch-${UPLOAD_CHANNEL}" pytorch "${cudatoolkit}"
32+
conda install -y -c "pytorch-${UPLOAD_CHANNEL}" ${CONDA_CHANNEL_FLAGS} pytorch "${cudatoolkit}"
2733

2834
printf "* Installing torchvision\n"
2935
python setup.py develop

‎.circleci/unittest/windows/scripts/environment.yml

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,18 +1,20 @@
11
channels:
22
- pytorch
33
- defaults
4+
# use conda-forge for python v3.9+
5+
- conda-forge
46
dependencies:
57
- numpy
68
- pytest
79
- pytest-cov
810
- codecov
911
- pip
1012
- libpng
11-
- jpeg
13+
- jpeg <= 9b
1214
- ca-certificates
15+
- av
1316
- pip:
1417
- future
1518
- pillow>=4.1.1
16-
- scipy
17-
- av
19+
- scipy>=1.4.1
1820
- dataclasses

‎.circleci/unittest/windows/scripts/install.sh

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,8 +24,14 @@ else
2424
version="$(python -c "print('.'.join(\"${CUDA_VERSION}\".split('.')[:2]))")"
2525
cudatoolkit="cudatoolkit=${version}"
2626
fi
27+
28+
CONDA_CHANNEL_FLAGS=""
29+
if [[ "$(python --version)" = *3.9* ]]; then
30+
CONDA_CHANNEL_FLAGS="-c=conda-forge"
31+
fi
32+
2733
printf "Installing PyTorch with %s\n" "${cudatoolkit}"
28-
conda install -y -c "pytorch-${UPLOAD_CHANNEL}" pytorch "${cudatoolkit}"
34+
conda install -y -c "pytorch-${UPLOAD_CHANNEL}" ${CONDA_CHANNEL_FLAGS} pytorch "${cudatoolkit}"
2935

3036
printf "* Installing torchvision\n"
3137
"$this_dir/vc_env_helper.bat" python setup.py develop

‎packaging/pkg_helpers.bash

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -179,10 +179,13 @@ setup_wheel_python() {
179179
if [[ "$(uname)" == Darwin || "$OSTYPE" == "msys" ]]; then
180180
eval "$(conda shell.bash hook)"
181181
conda env remove -n "env$PYTHON_VERSION" || true
182-
conda create -yn "env$PYTHON_VERSION" python="$PYTHON_VERSION"
182+
if [[ "$PYTHON_VERSION" == 3.9 ]]; then
183+
export CONDA_CHANNEL_FLAGS="${CONDA_CHANNEL_FLAGS} -c=conda-forge"
184+
fi
185+
conda create ${CONDA_CHANNEL_FLAGS} -yn "env$PYTHON_VERSION" python="$PYTHON_VERSION"
183186
conda activate "env$PYTHON_VERSION"
184187
# Install libpng from Anaconda (defaults)
185-
conda install libpng jpeg -y
188+
conda install ${CONDA_CHANNEL_FLAGS} -c conda-forge libpng "jpeg<=9b" -y
186189
else
187190
# Install native CentOS libJPEG, LAME, freetype and GnuTLS
188191
yum install -y libjpeg-turbo-devel lame freetype gnutls
@@ -198,6 +201,7 @@ setup_wheel_python() {
198201
3.6) python_abi=cp36-cp36m ;;
199202
3.7) python_abi=cp37-cp37m ;;
200203
3.8) python_abi=cp38-cp38 ;;
204+
3.9) python_abi=cp39-cp39 ;;
201205
*)
202206
echo "Unrecognized PYTHON_VERSION=$PYTHON_VERSION"
203207
exit 1
@@ -272,6 +276,9 @@ setup_conda_pytorch_constraint() {
272276
if [[ "$OSTYPE" == msys && "$CU_VERSION" == cu92 ]]; then
273277
export CONDA_CHANNEL_FLAGS="${CONDA_CHANNEL_FLAGS} -c defaults -c numba/label/dev"
274278
fi
279+
if [[ "$PYTHON_VERSION" == 3.9 ]]; then
280+
export CONDA_CHANNEL_FLAGS="${CONDA_CHANNEL_FLAGS} -c=conda-forge"
281+
fi
275282
}
276283

277284
# Translate CUDA_VERSION into CUDA_CUDATOOLKIT_CONSTRAINT

‎packaging/torchvision/meta.yaml

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ requirements:
99
build:
1010
- {{ compiler('c') }} # [win]
1111
- libpng
12-
- jpeg
12+
- jpeg <=9b
1313
- ffmpeg =4.2 # [not win]
1414

1515
host:
@@ -23,7 +23,7 @@ requirements:
2323
- python
2424
- libpng
2525
- ffmpeg =4.2 # [not win]
26-
- jpeg
26+
- jpeg <=9b
2727
- pillow >=4.1.1
2828
- numpy >=1.11
2929
{{ environ.get('CONDA_PYTORCH_CONSTRAINT') }}
@@ -50,7 +50,8 @@ test:
5050
requires:
5151
- pytest
5252
- scipy
53-
- av =8.0.1
53+
- av >=8.0.1
54+
- jpeg <=9b
5455
- ca-certificates
5556
{{ environ.get('CONDA_TYPING_CONSTRAINT') }}
5657

‎setup.py

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -330,7 +330,9 @@ def get_extensions():
330330
))
331331

332332
ffmpeg_exe = distutils.spawn.find_executable('ffmpeg')
333-
has_ffmpeg = ffmpeg_exe is not None
333+
# Disable ffmpeg by default
334+
no_ffmpeg = os.environ.get("NO_FFMPEG", True)
335+
has_ffmpeg = ffmpeg_exe is not None and not no_ffmpeg
334336
print("FFmpeg found: {}".format(has_ffmpeg))
335337

336338
if has_ffmpeg:

0 commit comments

Comments
 (0)