docs: record the kernel path is wired + e2e-covered; changelog the fixes #47
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # SPDX-License-Identifier: Apache-2.0 | |
| # | |
| # CI for pyspark-connect-web (the owns this). | |
| # | |
| # Jobs: | |
| # * unit - pytest across a Python matrix (transport stubbed; no | |
| # browser, no grpcio) | |
| # * lint - ruff check + format --check | |
| # * grpcio-guard - fails if grpcio is imported anywhere under | |
| # pyspark_connect_web/, package-scoped | |
| # * headers-guard - fails if deploy config drops the mandatory COOP/COEP | |
| # headers, or if prod CORS regresses to a wildcard | |
| #; runs scripts/validate_deploy.py | |
| # * build-wheel - builds the wheel and asserts it imports + carries no | |
| # grpcio dependency | |
| # | |
| # The e2e job is intentionally NOT a required gate yet: the browser stack | |
| # (the components + JupyterLite build) is not runnable in CI, so e2e would skip. A | |
| # commented sketch is included at the bottom to flip on once the stack lands. | |
| name: ci | |
| on: | |
| push: | |
| branches: [main] | |
| pull_request: | |
| jobs: | |
| unit: | |
| name: unit tests (pytest, py ${{ matrix.python-version }}) | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| python-version: ["3.11", "3.12"] | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: install | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install -e ".[dev]" | |
| - name: pytest (+ coverage) | |
| # Unit tests only. Excludes: | |
| # tests/e2e - Playwright/TS, needs a browser + live stack | |
| # tests/integration - needs a real Spark Connect server + real grpcio | |
| # (covered by the dedicated `integration` job below) | |
| # grpcio/grpcio-status are deliberately NOT installed here, so this job | |
| # also proves the package imports under the Pyodide constraint (_grpc_shim). | |
| run: > | |
| pytest -q --ignore=tests/e2e --ignore=tests/integration | |
| --cov=pyspark_connect_web --cov-report=term-missing | |
| --cov-report=xml --cov-fail-under=90 | |
| integration: | |
| name: integration (real Spark ${{ matrix.spark }} Connect server) | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| # Prove the >=4.0 claim against BOTH the 4.0 and 4.1 lines (matched | |
| # client+server pairs). Spark Connect's wire protocol is stable across | |
| # 4.x, so the pcw grpc-web client + framing must round-trip on each. | |
| # (4.1.0/4.1.1 were never published; 4.1.2 is the latest 4.1.x.) | |
| spark: ["4.0.0", "4.1.2"] | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-java@v4 | |
| with: | |
| distribution: temurin | |
| java-version: "17" | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.11" | |
| - name: install | |
| # The integration harness starts a real in-process Connect server (from | |
| # the pip pyspark, whose bundled Connect jar needs no extra download) and | |
| # uses a pure-Python grpc-web<->gRPC bridge that DOES need real grpcio + | |
| # grpcio-status - these live only in tests/ (the Envoy stand-in), which | |
| # is outside the package, so the no-grpcio-in-package rule still holds. | |
| # Pin pyspark to the matrix version so server AND client are that release. | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install -e ".[dev]" | |
| pip install "pyspark==${{ matrix.spark }}" grpcio grpcio-status | |
| - name: pytest tests/integration | |
| run: pytest -q tests/integration | |
| lint: | |
| name: lint + format (ruff) | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.11" | |
| - name: install ruff | |
| run: pip install ruff==0.6.9 | |
| # Scoped to scripts/ + the e2e reference generator for now; whole-repo lint | |
| # can be enabled once a [tool.ruff] config lands in pyproject.toml. | |
| - name: ruff check | |
| run: ruff check scripts/ tests/e2e/reference.py | |
| - name: ruff format --check | |
| run: ruff format --check scripts/ tests/e2e/reference.py | |
| js-bridge: | |
| name: js bridge unit tests (vitest) | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-node@v4 | |
| with: | |
| node-version: "20" | |
| - name: install + test | |
| # No lockfile committed; `npm install` resolves vitest at run time. | |
| run: cd tests/js && npm install && npm test | |
| grpcio-guard: | |
| name: no grpcio under pyspark_connect_web/ | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: grep for grpcio imports | |
| run: | | |
| set -euo pipefail | |
| # Match `import grpc`, `import grpcio`, `from grpc import ...`, | |
| # `from grpcio... import ...`. Scoped to the package only - tests/ | |
| # (e.g. the native reference generator) are allowed to use grpcio. | |
| if grep -rnE '^[[:space:]]*(import[[:space:]]+grpc(io)?\b|from[[:space:]]+grpc(io)?[._[:space:]])' \ | |
| pyspark_connect_web/ ; then | |
| echo "::error::grpcio/grpc imported inside pyspark_connect_web/ - forbidden." | |
| exit 1 | |
| fi | |
| echo "OK: no grpcio imports in pyspark_connect_web/" | |
| headers-guard: | |
| name: COOP/COEP + tight prod CORS in deploy config | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.11" | |
| - name: install pyyaml | |
| run: pip install pyyaml | |
| - name: validate deploy configs | |
| # Parses every Envoy/compose YAML, asserts COOP/COEP present in dev AND | |
| # prod, and that prod CORS has no wildcard origin. | |
| run: python scripts/validate_deploy.py | |
| build-wheel: | |
| name: build wheel + assert importable / no grpcio | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.11" | |
| - name: build | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install build==1.2.2 | |
| python -m build --wheel --outdir dist | |
| - name: install + import the wheel | |
| run: | | |
| set -euo pipefail | |
| pip install dist/pyspark_connect_web-*.whl | |
| python -c "import pyspark_connect_web; print('import OK', pyspark_connect_web.__name__)" | |
| - name: assert wheel declares no grpcio dependency | |
| run: | | |
| python - <<'PY' | |
| import importlib.metadata as m | |
| reqs = m.requires("pyspark-connect-web") or [] | |
| bad = [r for r in reqs if "grpcio" in r] | |
| assert not bad, f"grpcio in wheel requirements: {bad}" | |
| print("OK: no grpcio in wheel requirements") | |
| PY | |
| - uses: actions/upload-artifact@v4 | |
| with: | |
| name: wheel | |
| path: dist/*.whl | |
| # --------------------------------------------------------------------------- | |
| # e2e (DISABLED until the browser stack lands). Uncomment + set | |
| # E2E_REQUIRE_STACK=1 once the components + the JupyterLite build are runnable. | |
| # With the bridge wired, this asserts the full the v0 matrix matrix. | |
| # --------------------------------------------------------------------------- | |
| # e2e: | |
| # name: headless-browser e2e | |
| # runs-on: ubuntu-latest | |
| # steps: | |
| # - uses: actions/checkout@v4 | |
| # - name: build site + bring up stack | |
| # run: | | |
| # pip install -e ".[dev]" | |
| # scripts/build_site.sh | |
| # docker compose -f deploy/compose.yaml up -d | |
| # - name: reference results | |
| # run: python tests/e2e/reference.py --remote sc://localhost:15002 --out tests/e2e/reference.json | |
| # - uses: actions/setup-node@v4 | |
| # with: { node-version: "20" } | |
| # - name: playwright | |
| # working-directory: tests/e2e | |
| # run: | | |
| # npm install | |
| # npx playwright install --with-deps chromium | |
| # E2E_REQUIRE_STACK=1 E2E_BASE_URL=http://localhost:8000 npx playwright test |