Skip to content

Commit ede122d

Browse files
bagxibagxi
authored
on deploy action update (#1260)
* docker publisher action added * full images added * "build docs on publish" step added * versions updation with sed added * prepare "deploy_publish" action * CHANGELOG updated
1 parent 146885c commit ede122d

File tree

4 files changed

+186
-62
lines changed

4 files changed

+186
-62
lines changed

‎.github/workflows/deploy_publish.yml‎

Lines changed: 169 additions & 61 deletions
Original file line numberDiff line numberDiff line change
@@ -1,61 +1,169 @@
1-
#name: deploy-on-release
2-
#
3-
#on:
4-
# release:
5-
# types: [published]
6-
#
7-
#jobs:
8-
#
9-
# # build-pypi:
10-
# # runs-on: ubuntu-latest
11-
#
12-
# # steps:
13-
# # - uses: actions/checkout@v2
14-
#
15-
# # - name: set up Python 3.6
16-
# # uses: actions/setup-python@v1
17-
# # with:
18-
# # python-version: 3.6
19-
#
20-
# # - name: Generating distribution archives
21-
# # run: |
22-
# # pip install --upgrade setuptools wheel
23-
# # python setup.py sdist bdist_wheel --universal
24-
#
25-
# # - name: Publish a Python distribution to PyPI
26-
# # uses: pypa/gh-action-pypi-publish@master
27-
# # with:
28-
# # user: __token__
29-
# # password: ${{ secrets.PYPI_PASSWORD }}
30-
#
31-
# build-docker:
32-
# runs-on: ubuntu-latest
33-
# strategy:
34-
# matrix:
35-
# docker: ['', '-fp16', '-dev', '-dev-fp16']
36-
#
37-
# steps:
38-
# - uses: actions/checkout@v2
39-
#
40-
# - name: Get release version
41-
# id: get_version
42-
# run: echo "RELEASE_VERSION=$(echo ${GITHUB_REF:10})" >> ${GITHUB_ENV}
43-
#
44-
# - name: Publish Docker to Docker Hub
45-
# uses: elgohr/Publish-Docker-Github-Action@master
46-
# with:
47-
# name: catalystteam/catalyst
48-
# username: ${{ secrets.DOCKER_USERNAME }}
49-
# password: ${{ secrets.DOCKER_PASSWORD }}
50-
# dockerfile: docker/Dockerfile${{ matrix.docker }}
51-
# tags: "latest,${{ env.RELEASE_VERSION }}${{ matrix.docker }}"
52-
#
53-
# # build-docs:
54-
# # runs-on: ubuntu-latest
55-
#
56-
# # steps:
57-
# # - uses: actions/checkout@v2
58-
#
59-
# # - name: Get release version
60-
# # id: get_version
61-
# # run: echo ::set-env name=RELEASE_VERSION::$(echo ${GITHUB_REF:10})
1+
name: deploy-on-release
2+
3+
on:
4+
release:
5+
types: [published]
6+
7+
jobs:
8+
9+
# build-pypi:
10+
# runs-on: ubuntu-latest
11+
12+
# steps:
13+
# - uses: actions/checkout@v2
14+
15+
# - name: set up Python 3.6
16+
# uses: actions/setup-python@v1
17+
# with:
18+
# python-version: 3.6
19+
20+
# - name: Generating distribution archives
21+
# run: |
22+
# pip install --upgrade setuptools wheel
23+
# python setup.py sdist bdist_wheel --universal
24+
25+
# - name: Publish a Python distribution to PyPI
26+
# uses: pypa/gh-action-pypi-publish@master
27+
# with:
28+
# user: __token__
29+
# password: ${{ secrets.PYPI_PASSWORD }}
30+
31+
build-docker:
32+
runs-on: ubuntu-latest
33+
strategy:
34+
matrix:
35+
catalyst_requirements: ["base", all]
36+
pytorch:
37+
# - "1.9.0-cuda11.1-cudnn8-runtime"
38+
- "1.9.0-cuda11.1-cudnn8-devel"
39+
# - "1.8-cuda10.1-cudnn7-runtime"
40+
- "1.8.1-cuda11.1-cudnn8-devel"
41+
# - "1.7.0-cuda11.0-cudnn8-runtime"
42+
- "1.7.0-cuda11.0-cudnn8-devel"
43+
# - "1.3-cuda10.1-cudnn7-runtime"
44+
- "1.3-cuda10.1-cudnn7-devel"
45+
46+
steps:
47+
- uses: actions/checkout@v2
48+
49+
- name: Publish Docker to Docker Hub (base)
50+
if: ${{ matrix.catalyst_requirements == "base" }}
51+
uses: elgohr/Publish-Docker-Github-Action@master
52+
env:
53+
PYTORCH_TAG: "${{ matrix.pytorch }}"
54+
with:
55+
name: catalystteam/catalyst
56+
username: ${{ secrets.DOCKER_USERNAME }}
57+
password: ${{ secrets.DOCKER_PASSWORD }}
58+
dockerfile: docker/Dockerfile
59+
tags: "${{ github.event.release.tag_name }}-pytorch-${{ matrix.pytorch }}"
60+
61+
- name: Publish Docker to Docker Hub ([all])
62+
if: ${{ matrix.catalyst_requirements == "all" }}
63+
uses: elgohr/Publish-Docker-Github-Action@master
64+
env:
65+
PYTORCH_TAG: "${{ matrix.pytorch }}"
66+
CATALYST_ALL: "1"
67+
with:
68+
name: catalystteam/catalyst
69+
username: ${{ secrets.DOCKER_USERNAME }}
70+
password: ${{ secrets.DOCKER_PASSWORD }}
71+
dockerfile: docker/Dockerfile
72+
tags: "${{ github.event.release.tag_name }}-all-pytorch-${{ matrix.pytorch }}"
73+
74+
build-docs:
75+
name: build-docs
76+
runs-on: ${{ matrix.os }}
77+
strategy:
78+
fail-fast: false
79+
max-parallel: 4
80+
matrix:
81+
os: [ubuntu-18.04] # , windows-2019, macOS-10.15]
82+
python-version: [3.6] #, 3.7, 3.8]
83+
requires: ['latest']
84+
# exclude:
85+
# # excludes node 4 on macOS
86+
# - python-version: 3.8
87+
# requires: 'minimal'
88+
# Timeout: 4
89+
timeout-minutes: 30
90+
steps:
91+
- uses: actions/checkout@v2
92+
93+
- name: set up Python ${{ matrix.python-version }}
94+
uses: actions/setup-python@v2
95+
with:
96+
python-version: ${{ matrix.python-version }}
97+
98+
- name: setup ubuntu
99+
run: |
100+
python -c "from pip._internal.locations import USER_CACHE_DIR; print('::set-output name=dir::' + USER_CACHE_DIR)"
101+
102+
# https://github.com/actions/cache/blob/master/examples.md
103+
# Note: This uses an internal pip API and may not always work
104+
# https://github.com/actions/cache/blob/master/examples.md#multiple-oss-in-a-workflow
105+
- name: get pip cache dir
106+
id: pip-cache
107+
run: |
108+
python -c "from pip._internal.locations import USER_CACHE_DIR; print('::set-output name=dir::' + USER_CACHE_DIR)"
109+
110+
- name: cache pip
111+
uses: actions/cache@v2
112+
with:
113+
path: ${{ steps.pip-cache.outputs.dir }}
114+
key: ${{ runner.os }}-${{ matrix.python-version }}-pip -${{ hashFiles('./requirements/requirements.txt') }} -${{ hashFiles('./requirements/requirements-dev.txt') }}
115+
restore-keys: |
116+
${{ runner.os }}-${{ matrix.python-version }}-pip-
117+
118+
- name: cache pip contrib
119+
uses: actions/cache@v2
120+
with:
121+
path: ${{ steps.pip-cache.outputs.dir }}
122+
key: ${{ runner.os }}-${{ matrix.python-version }}-pip -${{hashFiles('./requirements/requirements-cv.txt') }} -${{hashFiles('./requirements/requirements-nifti.txt') }} -${{ hashFiles('./requirements/requirements-hydra.txt') }} -${{ hashFiles('./requirements/requirements-ml.txt') }} -${{ hashFiles('./requirements/requirements-optuna.txt') }} -${{ hashFiles('./requirements/requirements-mlflow.txt') }} -${{ hashFiles('./requirements/requirements-neptune.txt') }}
123+
restore-keys: |
124+
${{ runner.os }}-${{ matrix.python-version }}-pip-contrib
125+
126+
- name: cache pip contrib [2]
127+
uses: actions/cache@v2
128+
with:
129+
path: ${{ steps.pip-cache.outputs.dir }}
130+
key: ${{ runner.os }}-${{ matrix.python-version }}-pip -${{ hashFiles('./requirements/requirements-albu.txt') }}
131+
restore-keys: |
132+
${{ runner.os }}-${{ matrix.python-version }}-pip-contrib
133+
134+
- name: install dependencies
135+
run: |
136+
# python -m pip install --upgrade --user pip
137+
pip install -r ./requirements/requirements.txt -r ./requirements/requirements-cv.txt -r ./requirements/requirements-dev.txt -r ./requirements/requirements-hydra.txt -r ./requirements/requirements-ml.txt -r ./requirements/requirements-nifti.txt -r ./requirements/requirements-optuna.txt -r ./requirements/requirements-mlflow.txt -r ./requirements/requirements-neptune.txt -r ./requirements/requirements-albu.txt
138+
python --version
139+
pip --version
140+
pip list
141+
shell: bash
142+
143+
- name: make docs
144+
run: |
145+
REMOVE_BUILDS=0 make check-docs
146+
147+
- name: commit documentation changes
148+
env:
149+
TAG: "${{ github.event.release.tag_name }}"
150+
run: |
151+
git clone https://github.com/catalyst-team/catalyst.git --branch gh-pages --single-branch gh-pages
152+
cd gh-pages
153+
cp -a ../builds "${{ github.event.release.tag_name }}"
154+
# add link to new version into `versions.html`
155+
master_version_link='<li class="toctree-l1"><a class="reference internal" href="index.html">master<\/a><\/li>'
156+
new_version_link="\ <li class=\"toctree-l1\"><a class=\"reference internal\" href=\"${TAG}/index.html\">${TAG}<\/a><\/li>"
157+
sed -i "/${master_version_link}/a ${new_version_link}" versions.html
158+
# commit changes
159+
git config --local user.email "[email protected]"
160+
git config --local user.name "GitHub Action"
161+
git add .
162+
git commit -m "$(git log -1 --pretty=%B)" || true
163+
164+
- name: push changes
165+
uses: ad-m/github-push-action@master
166+
with:
167+
branch: gh-pages
168+
directory: gh-pages
169+
github_token: ${{ secrets.GITHUB_TOKEN }}

‎CHANGELOG.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/).
99
### Added
1010

1111
- added `pre-commit` hook to run codestyle checker on commit ([#1257](https://github.com/catalyst-team/catalyst/pull/1257))
12+
- `on publish` github action for docker and docs added [#1260](https://github.com/catalyst-team/catalyst/pull/1260)
1213

1314
### Changed
1415

‎docker/Dockerfile‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -44,10 +44,17 @@ RUN pip install -U catalyst --no-cache-dir
4444
RUN if [ "$CATALYST_DEV" = "1" ] ; then pip install -U catalyst[dev] --no-cache-dir ; else echo "catalyst[dev] is not required" ; fi
4545
RUN if [ "$CATALYST_CV" = "1" ] ; then pip install -U catalyst[cv] --no-cache-dir ; else echo "catalyst[cv] is not required" ; fi
4646
RUN if [ "$CATALYST_ML" = "1" ] ; then pip install -U catalyst[ml] --no-cache-dir ; else echo "catalyst[ml] is not required" ; fi
47-
RUN if [ "$CATALYST_ALBU" = "1" ] ; then pip install -U catalyst[albu] --no-cache-dir ; else echo "catalyst[albu] is not required" ; fi
47+
RUN if [ "$CATALYST_NIFTI" = "1" ] ; then pip install -U catalyst[nifti] --no-cache-dir ; else echo "catalyst[nifti] is not required" ; fi
4848
RUN if [ "$CATALYST_HYDRA" = "1" ] ; then pip install -U catalyst[hydra] --no-cache-dir ; else echo "catalyst[hydra] is not required" ; fi
4949
RUN if [ "$CATALYST_OPTUNA" = "1" ] ; then pip install -U catalyst[optuna] --no-cache-dir ; else echo "catalyst[optuna] is not required" ; fi
5050
RUN if [ "$CATALYST_ONNX" = "1" ] ; then pip install -U catalyst[onnx] --no-cache-dir ; else echo "catalyst[onnx] is not required" ; fi
5151
RUN if [ "$CATALYST_ONNX_GPU" = "1" ] ; then pip install -U catalyst[onnx-gpu] --no-cache-dir ; else echo "catalyst[onnx-gpu] is not required" ; fi
52+
RUN if [ "$CATALYST_MLFLOW" = "1" ] ; then pip install -U catalyst[mlflow] --no-cache-dir ; else echo "catalyst[mlflow] is not required" ; fi
53+
RUN if [ "$CATALYST_NEPTUNE" = "1" ] ; then pip install -U catalyst[neptune] --no-cache-dir ; else echo "catalyst[neptune] is not required" ; fi
54+
RUN if [ "$CATALYST_WANDB" = "1" ] ; then pip install -U catalyst[wandb] --no-cache-dir ; else echo "catalyst[wandb] is not required" ; fi
55+
RUN if [ "$CATALYST_FAIRSCALE" = "1" ] ; then pip install -U catalyst[fairscale] --no-cache-dir ; else echo "catalyst[fairscale] is not required" ; fi
56+
RUN if [ "$CATALYST_DEEPSPEED" = "1" ] ; then pip install -U catalyst[deepspeed] --no-cache-dir ; else echo "catalyst[deepspeed] is not required" ; fi
57+
RUN if [ "$CATALYST_ALBU" = "1" ] ; then pip install -U catalyst[albu] --no-cache-dir ; else echo "catalyst[albu] is not required" ; fi
58+
RUN if [ "$CATALYST_ALL" = "1" ] ; then pip install -U catalyst[all] --no-cache-dir ; else echo "catalyst[all] is not required" ; fi
5259

5360
WORKDIR /workspace

‎docker/Dockerfile-dev‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,9 +44,17 @@ COPY requirements/ /workspace/
4444
RUN if [ "$CATALYST_DEV" = "1" ] ; then pip install -r /workspace/requirements-dev.txt --no-cache-dir ; else echo "catalyst[dev] is not required" ; fi
4545
RUN if [ "$CATALYST_CV" = "1" ] ; then pip install -r /workspace/requirements-cv.txt --no-cache-dir ; else echo "catalyst[cv] is not required" ; fi
4646
RUN if [ "$CATALYST_ML" = "1" ] ; then pip install -r /workspace/requirements-ml.txt --no-cache-dir ; else echo "catalyst[ml] is not required" ; fi
47+
RUN if [ "$CATALYST_NIFTI" = "1" ] ; then pip install -r /workspace/requirements-nifti.txt --no-cache-dir ; else echo "catalyst[nifti] is not required" ; fi
4748
RUN if [ "$CATALYST_HYDRA" = "1" ] ; then pip install -r /workspace/requirements-hydra.txt --no-cache-dir ; else echo "catalyst[hydra] is not required" ; fi
4849
RUN if [ "$CATALYST_OPTUNA" = "1" ] ; then pip install -r /workspace/requirements-optuna.txt --no-cache-dir ; else echo "catalyst[optuna] is not required" ; fi
4950
RUN if [ "$CATALYST_ONNX" = "1" ] ; then pip install -r /workspace/requirements-onnx.txt --no-cache-dir ; else echo "catalyst[onnx] is not required" ; fi
5051
RUN if [ "$CATALYST_ONNX_GPU" = "1" ] ; then pip install -r /workspace/requirements-onnx-gpu.txt --no-cache-dir ; else echo "catalyst[onnx-gpu] is not required" ; fi
52+
RUN if [ "$CATALYST_MLFLOW" = "1" ] ; then pip install -r /workspace/requirements-mlflow.txt --no-cache-dir ; else echo "catalyst[mlflow] is not required" ; fi
53+
RUN if [ "$CATALYST_NEPTUNE" = "1" ] ; then pip install -r /workspace/requirements-neptune.txt --no-cache-dir ; else echo "catalyst[neptune] is not required" ; fi
54+
RUN if [ "$CATALYST_WANDB" = "1" ] ; then pip install -r /workspace/requirements-wandb.txt --no-cache-dir ; else echo "catalyst[wandb] is not required" ; fi
55+
RUN if [ "$CATALYST_FAIRSCALE" = "1" ] ; then pip install -r /workspace/requirements-fairscale.txt --no-cache-dir ; else echo "catalyst[fairscale] is not required" ; fi
56+
RUN if [ "$CATALYST_DEEPSPEED" = "1" ] ; then pip install -r /workspace/requirements-deepspeed.txt --no-cache-dir ; else echo "catalyst[deepspeed] is not required" ; fi
57+
RUN if [ "$CATALYST_ALBU" = "1" ] ; then pip install -r /workspace/requirements-albu.txt --no-cache-dir ; else echo "catalyst[albu] is not required" ; fi
58+
RUN if [ "$CATALYST_ALL" = "1" ] ; then pip install -r /workspace/requirements-all.txt --no-cache-dir ; else echo "catalyst[all] is not required" ; fi
5159

5260
WORKDIR /workspace

0 commit comments

Comments
 (0)