|
1 | 1 | import logging |
| 2 | +import os |
| 3 | +import re |
| 4 | +from collections.abc import Callable |
| 5 | +from datetime import datetime, timedelta, timezone |
| 6 | +from functools import partial |
2 | 7 | from unittest import skip |
3 | 8 | from unittest.mock import patch, create_autospec |
4 | 9 | import pytest |
@@ -609,3 +614,62 @@ def test_install_config_rejects_no_output_and_no_quarantine(ws): |
609 | 614 | match="At least one of an output table or a quarantine table", |
610 | 615 | ): |
611 | 616 | installer.configure() |
| 617 | + |
| 618 | + |
| 619 | +# DQX installs its workflows as jobs named "[<INSTALL_FOLDER>] <workflow>" (see mixins._get_name); |
| 620 | +# the workflow suffix is one of the deployed steps below. |
| 621 | +_DQX_JOB_NAME_PATTERN = re.compile(r"^\[.+\] (profiler|quality-checker|e2e|anomaly-trainer)$") |
| 622 | + |
| 623 | + |
| 624 | +def _safe_delete_job(delete: Callable[[], object], name: str) -> None: |
| 625 | + """Best-effort delete: a single failure is logged and never blocks the rest of the sweep.""" |
| 626 | + try: |
| 627 | + delete() |
| 628 | + except NotFound: |
| 629 | + pass |
| 630 | + except Exception as e: # noqa: BLE001 — best-effort sweep must not block the test run |
| 631 | + logger.warning(f"Failed to delete orphaned DQX job '{name}': {e}") |
| 632 | + |
| 633 | + |
| 634 | +def _remove_orphaned_jobs(ws) -> None: |
| 635 | + """Delete orphaned DQX workflow jobs (profiler, quality-checker, e2e, anomaly-trainer) left behind |
| 636 | + by cancelled CI runs. |
| 637 | +
|
| 638 | + Orphaned jobs accumulate when a github action is cancelled and the ``installation_ctx`` fixture |
| 639 | + teardown (which calls ``uninstall()`` to remove the deployed jobs) does not run. Jobs count toward |
| 640 | + the per-workspace job limit, so they are swept explicitly here, mirroring the orphaned-Lakebase |
| 641 | + sweep. |
| 642 | +
|
| 643 | + Runs only in CI. Jobs younger than the 2h grace period are skipped so a concurrent run's freshly |
| 644 | + deployed jobs (and the current run's own jobs) are never deleted; a real long-lived install is |
| 645 | + protected because the sweep only runs in the ephemeral CI test workspace. |
| 646 | + """ |
| 647 | + run_id = os.getenv("GITHUB_RUN_ID") |
| 648 | + if not run_id: |
| 649 | + return # only applicable when run in CI |
| 650 | + |
| 651 | + grace_period = datetime.now(timezone.utc) - timedelta(hours=2) # aligned with tests timeout |
| 652 | + |
| 653 | + matched = attempted = 0 |
| 654 | + for job in ws.jobs.list(): |
| 655 | + name = (job.settings.name if job.settings else None) or "" |
| 656 | + if not _DQX_JOB_NAME_PATTERN.match(name): |
| 657 | + continue |
| 658 | + matched += 1 |
| 659 | + created = datetime.fromtimestamp(job.created_time / 1000, tz=timezone.utc) if job.created_time else None |
| 660 | + if created is not None and created >= grace_period: |
| 661 | + continue # too young — may belong to a concurrent or the current run |
| 662 | + _safe_delete_job(partial(ws.jobs.delete, job_id=job.job_id), name) |
| 663 | + attempted += 1 |
| 664 | + logger.info(f"DQX job sweep: matched_naming={matched} delete_attempts={attempted}") |
| 665 | + |
| 666 | + |
| 667 | +def test_remove_orphaned_jobs(ws): |
| 668 | + """Maintenance sweep: remove orphaned DQX workflow jobs left by cancelled CI runs. |
| 669 | +
|
| 670 | + Mirrors the orphaned-Lakebase-resource sweep. Deployed workflow jobs (profiler, quality-checker, |
| 671 | + e2e, anomaly-trainer) count toward the per-workspace job limit and are only cleaned by the |
| 672 | + installation fixture teardown, so a cancelled run leaks them. Runs only in CI; jobs younger than |
| 673 | + the 2h grace period are skipped. |
| 674 | + """ |
| 675 | + _remove_orphaned_jobs(ws) |
0 commit comments