relax layer name constraints #5371
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI | |
| on: | |
| push: | |
| branches: | |
| - master | |
| tags: | |
| - "**" | |
| pull_request: {} | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: ${{ github.ref != 'refs/heads/master' }} | |
| jobs: | |
| changes: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| webknossos: ${{ steps.filter.outputs.webknossos || github.ref == 'refs/heads/master' }} | |
| cluster_tools: ${{ steps.filter.outputs.cluster_tools || github.ref == 'refs/heads/master' }} | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - uses: dorny/paths-filter@v2 | |
| id: filter | |
| with: | |
| filters: | | |
| webknossos: | |
| - 'webknossos/**' | |
| cluster_tools: | |
| - 'cluster_tools/**' | |
| cluster_tools_slurm: | |
| needs: changes | |
| if: ${{ needs.changes.outputs.cluster_tools == 'true' }} | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 30 | |
| strategy: | |
| matrix: | |
| python-version: ["3.13", "3.12", "3.11", "3.10"] | |
| defaults: | |
| run: | |
| working-directory: cluster_tools | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: astral-sh/setup-uv@v6 | |
| with: | |
| enable-cache: true | |
| cache-dependency-glob: "cluster_tools/uv.lock" | |
| - run: uv python install ${{ matrix.python-version }} | |
| - name: Start Docker Cluster | |
| run: cd ./dockered-slurm && docker compose up -d | |
| - name: Log Core Container | |
| run: | | |
| for name in "slurmctld" "c1" "c2"; do | |
| docker logs "$name" | |
| done | |
| - name: Install UV dependencies | |
| run: | | |
| for name in "slurmctld" "c1" "c2"; do | |
| docker exec -w /cluster_tools "$name" bash -c "uv sync --frozen" | |
| done | |
| - name: "Run Tests (test_all, test_slurm) without modified slurm.conf" | |
| run: | | |
| docker exec \ | |
| -w /cluster_tools/tests \ | |
| -e PYTEST_EXECUTORS=slurm \ | |
| slurmctld bash -c "uv run --frozen python -m pytest -sv test_all.py test_slurm.py -m 'not requires_modified_slurm_config'" | |
| - name: "Run Tests (test_deref_main)" | |
| run: | | |
| docker exec \ | |
| -w /cluster_tools/tests \ | |
| slurmctld bash -c "uv run --frozen python test_deref_main.py" | |
| - name: Update Slurm Config | |
| run: | | |
| echo "MaxArraySize=2" >> ./dockered-slurm/slurm.conf | |
| sed "s/JobAcctGatherFrequency=30/JobAcctGatherFrequency=1/g" ./dockered-slurm/slurm.conf > ./dockered-slurm/slurm.conf.tmp | |
| mv ./dockered-slurm/slurm.conf.tmp ./dockered-slurm/slurm.conf | |
| - name: Restart Slurm Cluster | |
| run: cd ./dockered-slurm && docker compose restart slurmctld c1 c2 | |
| - name: "Run Tests (test_all, test_slurm) with modified slurn.conf" | |
| run: | | |
| # Run tests requiring a modified slurm config | |
| docker exec \ | |
| -w /cluster_tools/tests \ | |
| -e PYTEST_EXECUTORS=slurm \ | |
| slurmctld bash -c "uv run --frozen python -m pytest -sv test_slurm.py -m 'requires_modified_slurm_config'" | |
| cluster_tools_multiprocessing: | |
| needs: changes | |
| if: ${{ needs.changes.outputs.cluster_tools == 'true' }} | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 30 | |
| strategy: | |
| matrix: | |
| python-version: ["3.13", "3.12", "3.11", "3.10"] | |
| defaults: | |
| run: | |
| working-directory: cluster_tools | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@v6 | |
| with: | |
| enable-cache: true | |
| cache-dependency-glob: "cluster_tools/uv.lock" | |
| - name: Set up Python ${{ matrix.python-version }} | |
| run: uv python install ${{ matrix.python-version }} | |
| - name: Install dependencies (without docker) | |
| run: uv sync --frozen | |
| - name: Check typing | |
| if: ${{ matrix.python-version == '3.11' }} | |
| run: ./typecheck.sh | |
| - name: Check formatting | |
| if: ${{ matrix.python-version == '3.11' }} | |
| run: ./format.sh check | |
| - name: Lint code | |
| if: ${{ matrix.python-version == '3.11' }} | |
| run: ./lint.sh | |
| - name: Run multiprocessing tests | |
| run: | | |
| cd tests | |
| PYTEST_EXECUTORS=multiprocessing,sequential,multiprocessing_with_pickling,sequential_with_pickling \ | |
| uv run --frozen python -m pytest -sv test_all.py test_multiprocessing.py | |
| webknossos_linux: | |
| needs: changes | |
| if: | | |
| ${{ needs.changes.outputs.cluster_tools == 'true' }} || | |
| ${{ needs.changes.outputs.webknossos == 'true' }} | |
| runs-on: ubuntu-latest | |
| strategy: | |
| matrix: | |
| python-version: ["3.13", "3.12", "3.11", "3.10"] | |
| group: [1, 2, 3] | |
| fail-fast: false | |
| defaults: | |
| run: | |
| working-directory: webknossos | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@v3 | |
| with: | |
| enable-cache: true | |
| cache-dependency-glob: "webknossos/uv.lock" | |
| - name: Install proxay | |
| run: npm install -g [email protected] | |
| - name: Set up Python ${{ matrix.python-version }} | |
| run: uv python install ${{ matrix.python-version }} | |
| - name: Check formatting | |
| if: ${{ matrix.group == 1 && matrix.python-version == '3.11' }} | |
| run: ./format.sh check | |
| - name: Lint code | |
| if: ${{ matrix.group == 1 && matrix.python-version == '3.11' }} | |
| run: ./lint.sh | |
| - name: Check typing | |
| if: ${{ matrix.group == 1 && matrix.python-version == '3.11' }} | |
| run: ./typecheck.sh | |
| - name: Patch Python standard library to assert that fork is not allowed | |
| run: | | |
| sed -i '/def _launch/a\ \ \ \ \ \ \ \ raise Exception("fork is not allowed.")' /home/runner/.local/share/uv/python/cpython-${{ matrix.python-version }}.*-linux-x86_64-gnu/lib/python${{ matrix.python-version }}/multiprocessing/popen_fork.py | |
| cat /home/runner/.local/share/uv/python/cpython-${{ matrix.python-version }}.*-linux-x86_64-gnu/lib/python${{ matrix.python-version }}/multiprocessing/popen_fork.py | |
| - name: Python tests (with coverage) | |
| if: ${{ matrix.python-version == '3.11' }} | |
| timeout-minutes: 30 | |
| env: | |
| PYTHON_VERSION: ${{ matrix.python-version }} | |
| run: uv run test.py --splits 3 --group ${{ matrix.group }} --splitting-algorithm least_duration --cov=webknossos --cov-report= | |
| - name: Python tests | |
| if: ${{ matrix.python-version != '3.11' }} | |
| timeout-minutes: 30 | |
| env: | |
| PYTHON_VERSION: ${{ matrix.python-version }} | |
| run: uv run test.py --splits 3 --group ${{ matrix.group }} --splitting-algorithm least_duration | |
| - name: Persist coverage reports | |
| if: ${{ matrix.python-version == '3.11' }} | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: coverage-report-${{ matrix.group }} | |
| path: "webknossos/.coverage" | |
| include-hidden-files: true | |
| if-no-files-found: error | |
| retention-days: 1 # Automatically deletes after 1 day | |
| - name: Check if git is dirty | |
| run: ../_tooling/check_clean.sh | |
| webknossos_windows: | |
| needs: changes | |
| if: | | |
| ${{ needs.changes.outputs.cluster_tools == 'true' }} || | |
| ${{ needs.changes.outputs.webknossos == 'true' }} | |
| runs-on: windows-latest | |
| strategy: | |
| matrix: | |
| python-version: ["3.12"] | |
| group: [1, 2, 3] | |
| fail-fast: false | |
| defaults: | |
| run: | |
| working-directory: webknossos | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@v3 | |
| with: | |
| enable-cache: true | |
| cache-dependency-glob: "webknossos/uv.lock" | |
| - name: Install minio | |
| run: Invoke-WebRequest -Uri "https://dl.min.io/server/minio/release/windows-amd64/minio.exe" -OutFile "minio.exe" | |
| - name: Install proxay | |
| run: npm install -g [email protected] | |
| - name: Set up Python ${{ matrix.python-version }} | |
| run: uv python install ${{ matrix.python-version }} | |
| - name: Python tests | |
| timeout-minutes: 30 | |
| env: | |
| PYTHON_VERSION: ${{ matrix.python-version }} | |
| MULTIPROCESSING_DEFAULT_START_METHOD: spawn | |
| run: uv run test.py --splits 3 --group ${{ matrix.group }} --splitting-algorithm least_duration | |
| coverage_report: | |
| needs: [webknossos_linux] | |
| if: success() | |
| runs-on: ubuntu-latest | |
| defaults: | |
| run: | |
| working-directory: webknossos | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Install coverage.py | |
| run: pip install coverage | |
| - name: Download all coverage reports | |
| uses: actions/download-artifact@v4 | |
| with: | |
| path: ~/coverage-files | |
| - name: Generate coverage report | |
| env: | |
| COVERAGE_FILE: "~/coverage-files/result.coverage" | |
| run: | | |
| coverage combine ~/coverage-files/**/.coverage | |
| coverage xml -i -o ~/coverage-files/result.xml | |
| - name: Report coverage | |
| uses: orgoro/[email protected] | |
| with: | |
| coverageFile: /home/runner/coverage-files/result.xml | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| thresholdAll: 0.8 | |
| - name: Cleanup temporary files | |
| run: rm -rf ~/coverage-files | |
| docs: | |
| needs: | |
| - cluster_tools_slurm | |
| - cluster_tools_multiprocessing | |
| - webknossos_linux | |
| - webknossos_windows | |
| runs-on: ubuntu-latest | |
| if: | | |
| always() && | |
| !contains(needs.*.result, 'failure') && | |
| !contains(needs.*.result, 'cancelled') && | |
| !github.event.pull_request.head.repo.fork | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - uses: actions/checkout@v3 | |
| with: | |
| repository: scalableminds/webknossos | |
| path: docs/wk-repo | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@v3 | |
| - name: Build Docs | |
| run: | | |
| cd docs | |
| ./generate.sh --persist | |
| - name: Push docs (for master) | |
| if: github.ref == 'refs/heads/master' | |
| env: | |
| AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| AWS_DEFAULT_REGION: "eu-west-1" | |
| run: | | |
| CI_BRANCH=${GITHUB_HEAD_REF:-$GITHUB_REF_NAME} | |
| NORMALIZED_CI_BRANCH=${CI_BRANCH//[\/-]/_} | |
| aws s3 sync --acl public-read docs/out s3://static.webknossos.org/docs/${NORMALIZED_CI_BRANCH} | |
| - name: Push docs (for tag) | |
| if: github.ref_type == 'tag' | |
| env: | |
| AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| AWS_DEFAULT_REGION: "eu-west-1" | |
| run: | | |
| CI_TAG=$(git describe --tags) | |
| aws s3 sync --acl public-read docs/out s3://static.webknossos.org/docs/${CI_TAG} | |
| - name: Check links (on master) | |
| if: github.ref == 'refs/heads/master' | |
| env: # Or as an environment variable | |
| SLACK_HOOK: ${{ secrets.LINK_CHECKER_SLACK_HOOK }} | |
| run: | | |
| cd docs | |
| uv run --frozen linkchecker --config linkcheckerrc https://docs.webknossos.org > link_status || \ | |
| curl -X POST --data-urlencode "payload={\"text\": \":warning: Broken Links on doc.webknossos.org :warning:\n"'```'"\n$(cat link_status)\n"'```"}' \ | |
| "$SLACK_HOOK" | |
| pypi_and_gh_release: | |
| needs: | |
| - cluster_tools_slurm | |
| - cluster_tools_multiprocessing | |
| - webknossos_linux | |
| - webknossos_windows | |
| if: | | |
| always() && | |
| !contains(needs.*.result, 'failure') && | |
| !contains(needs.*.result, 'cancelled') && | |
| github.ref_type == 'tag' && | |
| !github.event.pull_request.head.repo.fork | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@v3 | |
| - name: Publish python packages | |
| env: | |
| UV_PUBLISH_TOKEN: ${{ secrets.PYPI_PASSWORD }} | |
| run: _tooling/publish.sh | |
| - name: Prepare github release | |
| run: | | |
| VERSION="$(uvx dunamai from git)" | |
| _tooling/changelog_for_version.sh $VERSION > Changelog.md | |
| - name: Publish github release | |
| id: create_release | |
| uses: actions/create-release@v1 | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| with: | |
| tag_name: ${{ github.ref }} | |
| release_name: ${{ github.ref }} | |
| body_path: Changelog.md | |
| draft: false | |
| prerelease: false | |
| complete: | |
| needs: | |
| - cluster_tools_multiprocessing | |
| - cluster_tools_slurm | |
| - webknossos_linux | |
| - webknossos_windows | |
| - docs | |
| - pypi_and_gh_release | |
| if: always() | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Check failure | |
| if: | | |
| contains(needs.*.result, 'failure') || | |
| contains(needs.*.result, 'cancelled') | |
| run: exit 1 | |
| - name: Success | |
| run: echo Success! |