From c3d518ff11dd225dfcf195317e3bdbdcbeb028be Mon Sep 17 00:00:00 2001 From: Greg Hansen Date: Sat, 13 Sep 2025 13:41:04 -0400 Subject: [PATCH 01/31] Allow installation in a custom folder --- docs/dqx/docs/installation.mdx | 23 +++++++++++---- docs/dqx/docs/reference/cli.mdx | 5 ++-- src/databricks/labs/dqx/base.py | 2 +- src/databricks/labs/dqx/checks_storage.py | 6 ++-- src/databricks/labs/dqx/config.py | 2 ++ src/databricks/labs/dqx/config_loader.py | 18 +++++++++--- src/databricks/labs/dqx/engine.py | 8 +++-- .../labs/dqx/installer/config_provider.py | 8 ++++- src/databricks/labs/dqx/installer/install.py | 23 +++++++++++++-- tests/integration/test_installation.py | 29 +++++++++++++++++++ 10 files changed, 103 insertions(+), 21 deletions(-) diff --git a/docs/dqx/docs/installation.mdx b/docs/dqx/docs/installation.mdx index e3a02d244..97e851833 100644 --- a/docs/dqx/docs/installation.mdx +++ b/docs/dqx/docs/installation.mdx @@ -98,18 +98,29 @@ It is recommended to use serverless clusters for the workflows, as it allows for If serverless clusters are not used, a default cluster configuration will be used for the workflows. Alternatively, you can override the cluster configuration for each workflow in the `config.yml` file after the installation, to provide existing clusters to use. -#### User vs Global Installation +#### Installation Options -DQX is installed by default in the user home directory (under `/Users//.dqx`). You can also install DQX globally -by setting the 'DQX_FORCE_INSTALL' environment variable. The following options are available: +DQX offers flexible installation options. By default, DQX is installed in the user home directory (under `/Users//.dqx`). -* `DQX_FORCE_INSTALL=global databricks labs install dqx`: will force the installation to be for root only (`/Applications/dqx`) -* `DQX_FORCE_INSTALL=user databricks labs install dqx`: will force the installation to be for user only (`/Users//.dqx`) +**Custom Workspace Folder:** +If you provide a custom path during installation (e.g., `/Shared/dqx-team` or `/Users/shared-user/dqx-project`), DQX will be installed there. + +**Environment Variable Override:** +You can also force global or user installation using the 'DQX_FORCE_INSTALL' environment variable: + * `DQX_FORCE_INSTALL=global databricks labs install dqx`: forces installation to `/Applications/dqx` + * `DQX_FORCE_INSTALL=user databricks labs install dqx`: forces installation to `/Users//.dqx` + + +If a custom folder is provided during installation, the installation folder will take precedence over any environment variables (e.g. `DQX_FORCE_INSTALL`). + #### Configuration file DQX configuration file can contain multiple run configurations for different pipelines or projects, each defining specific input, output and quarantine locations, etc. -By default, the config is created in the installation directory under `/Users//.dqx/config.yml` or `/Applications/dqx/config.yml` if installed globally. +The configuration file is created in the installation directory: +- Custom folder: `/config.yml` +- User home (default): `/Users//.dqx/config.yml` +- Global: `/Applications/dqx/config.yml` The "default" run configuration is created during the installation. When DQX is upgraded, the configuration is preserved. The configuration can be updated / extended manually by the user after the installation. Each run config defines configuration for one specific input and output location. diff --git a/docs/dqx/docs/reference/cli.mdx b/docs/dqx/docs/reference/cli.mdx index 6e9483932..5c885c4f7 100644 --- a/docs/dqx/docs/reference/cli.mdx +++ b/docs/dqx/docs/reference/cli.mdx @@ -24,8 +24,9 @@ databricks labs uninstall dqx ``` -By default, DQX is installed under the user's home (for example `/Users//.dqx`). -Use the `DQX_FORCE_INSTALL` env var to force a global or user install. See the installation guide for details. +By default, DQX is installed under the user's home folder (for example `/Users//.dqx`). +Use the `DQX_FORCE_INSTALL` env var to force a global or user install. Provide a custom installation +folder during installation to override the default location. See the installation guide for details. ## Configuration helpers diff --git a/src/databricks/labs/dqx/base.py b/src/databricks/labs/dqx/base.py index 4bf3b88e9..ede1b32ba 100644 --- a/src/databricks/labs/dqx/base.py +++ b/src/databricks/labs/dqx/base.py @@ -45,7 +45,7 @@ def _verify_workspace_client(ws: WorkspaceClient) -> WorkspaceClient: setattr(ws.config, '_product_info', ('dqx', __version__)) # make sure Databricks workspace is accessible - ws.current_user.me() + ws.get_workspace_id() return ws diff --git a/src/databricks/labs/dqx/checks_storage.py b/src/databricks/labs/dqx/checks_storage.py index 65b65fc0b..d9494ca2f 100644 --- a/src/databricks/labs/dqx/checks_storage.py +++ b/src/databricks/labs/dqx/checks_storage.py @@ -254,9 +254,11 @@ def _get_storage_handler_and_config( self, config: InstallationChecksStorageConfig ) -> tuple[ChecksStorageHandler, InstallationChecksStorageConfig]: run_config = self._run_config_loader.load_run_config( - config.run_config_name, config.assume_user, config.product_name + run_config_name=config.run_config_name, assume_user=config.assume_user, product_name=config.product_name + ) + installation = self._run_config_loader.get_installation( + config.assume_user, config.product_name, config.install_folder ) - installation = self._run_config_loader.get_installation(config.assume_user, config.product_name) config.location = run_config.checks_location diff --git a/src/databricks/labs/dqx/config.py b/src/databricks/labs/dqx/config.py index 4eaf67a83..afacb08d1 100644 --- a/src/databricks/labs/dqx/config.py +++ b/src/databricks/labs/dqx/config.py @@ -219,9 +219,11 @@ class InstallationChecksStorageConfig( run_config_name: The name of the run configuration to use for checks (default is 'default'). product_name: The product name for retrieving checks from the installation (default is 'dqx'). assume_user: Whether to assume the user is the owner of the checks (default is True). + install_folder: The installation folder where DQX is installed """ location: str = "installation" # retrieved from the installation config run_config_name: str = "default" # to retrieve run config product_name: str = "dqx" assume_user: bool = True + install_folder: str | None = None diff --git a/src/databricks/labs/dqx/config_loader.py b/src/databricks/labs/dqx/config_loader.py index b42d7fd0f..dad6e8b0f 100644 --- a/src/databricks/labs/dqx/config_loader.py +++ b/src/databricks/labs/dqx/config_loader.py @@ -13,28 +13,38 @@ def __init__(self, workspace_client: WorkspaceClient): self.ws = workspace_client def load_run_config( - self, run_config_name: str | None, assume_user: bool = True, product_name: str = "dqx" + self, + run_config_name: str | None, + install_folder: str | None = None, + assume_user: bool = True, + product_name: str = "dqx", ) -> RunConfig: """ Load run configuration from the installation. Args: run_config_name: name of the run configuration to use + install_folder: optional installation folder assume_user: if True, assume user installation product_name: name of the product """ - installation = self.get_installation(assume_user, product_name) + installation = self.get_installation(assume_user, product_name, install_folder) return self._load_run_config(installation, run_config_name) - def get_installation(self, assume_user: bool, product_name: str) -> Installation: + def get_installation(self, assume_user: bool, product_name: str, install_folder: str | None = None) -> Installation: """ Get the installation for the given product name. Args: assume_user: if True, assume user installation product_name: name of the product + install_folder: optional installation folder """ - if assume_user: + + if install_folder: + installation = Installation(self.ws, product_name, install_folder=install_folder) + assume_user = False + elif assume_user: installation = Installation.assume_user_home(self.ws, product_name) else: installation = Installation.assume_global(self.ws, product_name) diff --git a/src/databricks/labs/dqx/engine.py b/src/databricks/labs/dqx/engine.py index 5734f99ac..57ebc9789 100644 --- a/src/databricks/labs/dqx/engine.py +++ b/src/databricks/labs/dqx/engine.py @@ -620,11 +620,15 @@ def save_results_in_table( None """ if output_df is not None and output_config is None: - run_config = self._run_config_loader.load_run_config(run_config_name, assume_user, product_name) + run_config = self._run_config_loader.load_run_config( + run_config_name=run_config_name, assume_user=assume_user, product_name=product_name + ) output_config = run_config.output_config if quarantine_df is not None and quarantine_config is None: - run_config = self._run_config_loader.load_run_config(run_config_name, assume_user, product_name) + run_config = self._run_config_loader.load_run_config( + run_config_name=run_config_name, assume_user=assume_user, product_name=product_name + ) quarantine_config = run_config.quarantine_config if output_df is not None and output_config is not None: diff --git a/src/databricks/labs/dqx/installer/config_provider.py b/src/databricks/labs/dqx/installer/config_provider.py index 9be82f6e5..575ee3a05 100644 --- a/src/databricks/labs/dqx/installer/config_provider.py +++ b/src/databricks/labs/dqx/installer/config_provider.py @@ -17,12 +17,18 @@ def __init__(self, prompts: Prompts, warehouse_configurator: WarehouseInstaller) self._prompts = prompts self._warehouse_configurator = warehouse_configurator - def prompt_new_installation(self) -> WorkspaceConfig: + def prompt_new_installation(self, install_folder: str | None = None) -> WorkspaceConfig: logger.info( "Please answer a couple of questions to provide default DQX run configuration. " "The configuration can also be updated manually after the installation." ) + # Show installation folder information + if install_folder: + logger.info(f"DQX will be installed in folder '{install_folder}'") + else: + logger.info("DQX will be installed in the default location (user home or global based on environment)") + log_level = self._prompts.question("Log level", default="INFO").upper() is_streaming = self._prompts.confirm("Should the input data be read using streaming?") input_config = self._prompt_input_config(is_streaming) diff --git a/src/databricks/labs/dqx/installer/install.py b/src/databricks/labs/dqx/installer/install.py index a0d8d3b4d..d62730bfa 100644 --- a/src/databricks/labs/dqx/installer/install.py +++ b/src/databricks/labs/dqx/installer/install.py @@ -45,14 +45,16 @@ class WorkspaceInstaller(WorkspaceContext): Args: environ: Optional dictionary of environment variables. ws: The WorkspaceClient instance. + install_folder: Optional custom workspace folder path for installation. """ - def __init__(self, ws: WorkspaceClient, environ: dict[str, str] | None = None): + def __init__(self, ws: WorkspaceClient, environ: dict[str, str] | None = None, install_folder: str | None = None): super().__init__(ws) if not environ: environ = dict(os.environ.items()) self._force_install = environ.get("DQX_FORCE_INSTALL") + self._install_folder = install_folder if "DATABRICKS_RUNTIME_VERSION" in environ: msg = "WorkspaceInstaller is not supposed to be executed in Databricks Runtime" @@ -82,6 +84,10 @@ def installation(self): try: return self.product_info.current_installation(self.workspace_client) except NotFound: + if self._install_folder: + return Installation( + self.workspace_client, self.product_info.product_name(), install_folder=self._install_folder + ) if self._force_install == "global": return Installation.assume_global(self.workspace_client, self.product_info.product_name()) return Installation.assume_user_home(self.workspace_client, self.product_info.product_name()) @@ -204,7 +210,7 @@ def _is_testing(self): def _prompt_for_new_installation(self) -> WorkspaceConfig: configurator = WarehouseInstaller(self.workspace_client, self.prompts) prompter = ConfigProvider(self.prompts, configurator) - return prompter.prompt_new_installation() + return prompter.prompt_new_installation(self._install_folder) def _confirm_force_install(self) -> bool: if not self._force_install: @@ -366,5 +372,16 @@ def _create_dashboard(self) -> None: if is_in_debug(): logging.getLogger("databricks").setLevel(logging.DEBUG) - workspace_installer = WorkspaceInstaller(WorkspaceClient(product="dqx", product_version=__version__)) + installer_prompts = Prompts() + custom_folder = installer_prompts.question( + "Enter a workspace path for DQX installation (leave empty for default behavior):", + default="", + valid_regex=r"^(/.*)?$", + ) + + custom_install_folder = custom_folder.strip() if custom_folder.strip() else None + + workspace_installer = WorkspaceInstaller( + WorkspaceClient(product="dqx", product_version=__version__), install_folder=custom_install_folder + ) workspace_installer.run() diff --git a/tests/integration/test_installation.py b/tests/integration/test_installation.py index 8a767091b..f12b3abba 100644 --- a/tests/integration/test_installation.py +++ b/tests/integration/test_installation.py @@ -352,6 +352,35 @@ def test_workflows_deployment_creates_jobs_with_remove_after_tag(): wheels.assert_not_called() +def test_custom_folder_installation(ws, new_installation, make_random): + product_info = ProductInfo.for_testing(WorkspaceConfig) + custom_folder = f"/Shared/dqx-test-{make_random}" + + custom_installation = Installation(ws, product_info.product_name(), install_folder=custom_folder) + installation = new_installation( + product_info=product_info, + installation=custom_installation, + ) + + assert installation.install_folder() == custom_folder + assert ws.workspace.get_status(custom_folder) + + +def test_custom_folder_installation_with_environment_variable(ws, new_installation, make_random): + product_info = ProductInfo.for_testing(WorkspaceConfig) + custom_folder = f"/Shared/dqx-test-{make_random}" + + custom_installation = Installation(ws, product_info.product_name(), install_folder=custom_folder) + installation = new_installation( + product_info=product_info, + installation=custom_installation, + environ={'DQX_FORCE_INSTALL': 'global'}, # environment variable should not override the install folder + ) + + assert installation.install_folder() == custom_folder + assert ws.workspace.get_status(custom_folder) + + def test_my_username(): """Test the _my_username property to cover both conditions.""" From 7bde7cbd78f63b55eabc9fffde7d641076a5d776 Mon Sep 17 00:00:00 2001 From: Greg Hansen Date: Sun, 14 Sep 2025 08:41:17 -0400 Subject: [PATCH 02/31] Fix tests --- tests/integration/test_installation.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/integration/test_installation.py b/tests/integration/test_installation.py index f12b3abba..88d986783 100644 --- a/tests/integration/test_installation.py +++ b/tests/integration/test_installation.py @@ -352,9 +352,9 @@ def test_workflows_deployment_creates_jobs_with_remove_after_tag(): wheels.assert_not_called() -def test_custom_folder_installation(ws, new_installation, make_random): +def test_custom_folder_installation(ws, new_installation, make_directory): product_info = ProductInfo.for_testing(WorkspaceConfig) - custom_folder = f"/Shared/dqx-test-{make_random}" + custom_folder = make_directory().as_fuse() custom_installation = Installation(ws, product_info.product_name(), install_folder=custom_folder) installation = new_installation( @@ -366,9 +366,9 @@ def test_custom_folder_installation(ws, new_installation, make_random): assert ws.workspace.get_status(custom_folder) -def test_custom_folder_installation_with_environment_variable(ws, new_installation, make_random): +def test_custom_folder_installation_with_environment_variable(ws, new_installation, make_directory): product_info = ProductInfo.for_testing(WorkspaceConfig) - custom_folder = f"/Shared/dqx-test-{make_random}" + custom_folder = make_directory().as_fuse() custom_installation = Installation(ws, product_info.product_name(), install_folder=custom_folder) installation = new_installation( From 0579b51430e47b488a8a4c752ed55914868c939e Mon Sep 17 00:00:00 2001 From: Greg Hansen Date: Sun, 14 Sep 2025 11:55:10 -0400 Subject: [PATCH 03/31] Fix installation with custom folders --- src/databricks/labs/dqx/config_loader.py | 5 ++--- src/databricks/labs/dqx/installer/install.py | 7 +++--- src/databricks/labs/dqx/utils.py | 23 ++++++++++++++++++++ tests/integration/test_installation.py | 1 - 4 files changed, 29 insertions(+), 7 deletions(-) diff --git a/src/databricks/labs/dqx/config_loader.py b/src/databricks/labs/dqx/config_loader.py index dad6e8b0f..21bd37ee8 100644 --- a/src/databricks/labs/dqx/config_loader.py +++ b/src/databricks/labs/dqx/config_loader.py @@ -2,6 +2,7 @@ from databricks.sdk import WorkspaceClient from databricks.labs.dqx.config import RunConfig, WorkspaceConfig +from databricks.labs.dqx.utils import get_custom_installation class RunConfigLoader: @@ -42,14 +43,12 @@ def get_installation(self, assume_user: bool, product_name: str, install_folder: """ if install_folder: - installation = Installation(self.ws, product_name, install_folder=install_folder) - assume_user = False + installation = get_custom_installation(self.ws, product_name, install_folder) elif assume_user: installation = Installation.assume_user_home(self.ws, product_name) else: installation = Installation.assume_global(self.ws, product_name) - # verify the installation installation.current(self.ws, product_name, assume_user=assume_user) return installation diff --git a/src/databricks/labs/dqx/installer/install.py b/src/databricks/labs/dqx/installer/install.py index d62730bfa..e85691616 100644 --- a/src/databricks/labs/dqx/installer/install.py +++ b/src/databricks/labs/dqx/installer/install.py @@ -32,6 +32,7 @@ from databricks.labs.dqx.__about__ import __version__ from databricks.labs.dqx.config import WorkspaceConfig from databricks.labs.dqx.contexts.workspace_context import WorkspaceContext +from databricks.labs.dqx.utils import get_custom_installation logger = logging.getLogger(__name__) @@ -85,8 +86,8 @@ def installation(self): return self.product_info.current_installation(self.workspace_client) except NotFound: if self._install_folder: - return Installation( - self.workspace_client, self.product_info.product_name(), install_folder=self._install_folder + return get_custom_installation( + self.workspace_client, self.product_info.product_name(), self._install_folder ) if self._force_install == "global": return Installation.assume_global(self.workspace_client, self.product_info.product_name()) @@ -374,7 +375,7 @@ def _create_dashboard(self) -> None: installer_prompts = Prompts() custom_folder = installer_prompts.question( - "Enter a workspace path for DQX installation (leave empty for default behavior):", + "Enter a workspace path for DQX installation (leave empty for default behavior)", default="", valid_regex=r"^(/.*)?$", ) diff --git a/src/databricks/labs/dqx/utils.py b/src/databricks/labs/dqx/utils.py index a6424fc17..43d430865 100644 --- a/src/databricks/labs/dqx/utils.py +++ b/src/databricks/labs/dqx/utils.py @@ -8,6 +8,8 @@ from pyspark.sql.dataframe import DataFrame from pyspark.sql.connect.column import Column as ConnectColumn from databricks.labs.dqx.config import InputConfig, OutputConfig +from databricks.labs.blueprint.installation import Installation, NotInstalled +from databricks.sdk import WorkspaceClient logger = logging.getLogger(__name__) @@ -326,3 +328,24 @@ def safe_json_load(value: str): return json.loads(value) # load as json if possible except json.JSONDecodeError: return value + + +def get_custom_installation(ws: WorkspaceClient, product_name: str, install_folder: str) -> Installation: + """ + Creates an Installation instance for a custom folder, similar to assume_user_home and assume_global. + This ensures the custom folder is created in the workspace when the installation is accessed. + + Args: + ws: Databricks SDK `WorkspaceClient` + product_name: The product name + install_folder: The custom installation folder path + + Returns: + An Installation instance for the custom folder + """ + try: + ws.workspace.get_status(install_folder) + except NotInstalled: + ws.workspace.mkdirs(install_folder) + + return Installation(ws, product_name, install_folder=install_folder) diff --git a/tests/integration/test_installation.py b/tests/integration/test_installation.py index 88d986783..3ef3a1959 100644 --- a/tests/integration/test_installation.py +++ b/tests/integration/test_installation.py @@ -59,7 +59,6 @@ def factory( prompts=prompts, ) workspace_config = installer.configure() - installation = product_info.current_installation(ws) installation.save(workspace_config) cleanup.append(installation) return installation From 4e838abb0f3dfe0b1359c162d68b8ecbf354e741 Mon Sep 17 00:00:00 2001 From: Greg Hansen Date: Sun, 14 Sep 2025 12:01:48 -0400 Subject: [PATCH 04/31] Fix tests --- tests/integration/test_installation.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/integration/test_installation.py b/tests/integration/test_installation.py index 3ef3a1959..e89f1cc9b 100644 --- a/tests/integration/test_installation.py +++ b/tests/integration/test_installation.py @@ -353,7 +353,7 @@ def test_workflows_deployment_creates_jobs_with_remove_after_tag(): def test_custom_folder_installation(ws, new_installation, make_directory): product_info = ProductInfo.for_testing(WorkspaceConfig) - custom_folder = make_directory().as_fuse() + custom_folder = make_directory().as_fuse().as_posix() custom_installation = Installation(ws, product_info.product_name(), install_folder=custom_folder) installation = new_installation( @@ -367,7 +367,7 @@ def test_custom_folder_installation(ws, new_installation, make_directory): def test_custom_folder_installation_with_environment_variable(ws, new_installation, make_directory): product_info = ProductInfo.for_testing(WorkspaceConfig) - custom_folder = make_directory().as_fuse() + custom_folder = make_directory().as_fuse().as_posix() custom_installation = Installation(ws, product_info.product_name(), install_folder=custom_folder) installation = new_installation( From 462b1604e3538e2e7a0a19a71a69a23295eb32cf Mon Sep 17 00:00:00 2001 From: Greg Hansen Date: Sun, 14 Sep 2025 17:15:32 -0400 Subject: [PATCH 05/31] Fix tests --- tests/integration/test_installation.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/integration/test_installation.py b/tests/integration/test_installation.py index e89f1cc9b..9dc0641b6 100644 --- a/tests/integration/test_installation.py +++ b/tests/integration/test_installation.py @@ -353,7 +353,7 @@ def test_workflows_deployment_creates_jobs_with_remove_after_tag(): def test_custom_folder_installation(ws, new_installation, make_directory): product_info = ProductInfo.for_testing(WorkspaceConfig) - custom_folder = make_directory().as_fuse().as_posix() + custom_folder = str(make_directory().absolute()) custom_installation = Installation(ws, product_info.product_name(), install_folder=custom_folder) installation = new_installation( @@ -367,7 +367,7 @@ def test_custom_folder_installation(ws, new_installation, make_directory): def test_custom_folder_installation_with_environment_variable(ws, new_installation, make_directory): product_info = ProductInfo.for_testing(WorkspaceConfig) - custom_folder = make_directory().as_fuse().as_posix() + custom_folder = str(make_directory().absolute()) custom_installation = Installation(ws, product_info.product_name(), install_folder=custom_folder) installation = new_installation( From 6fc83a0624f94c6e89409b16578078e8cd0f8f00 Mon Sep 17 00:00:00 2001 From: Greg Hansen Date: Mon, 15 Sep 2025 08:04:41 -0400 Subject: [PATCH 06/31] Fix handling of new paths --- src/databricks/labs/dqx/utils.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/databricks/labs/dqx/utils.py b/src/databricks/labs/dqx/utils.py index 43d430865..68a809634 100644 --- a/src/databricks/labs/dqx/utils.py +++ b/src/databricks/labs/dqx/utils.py @@ -10,6 +10,7 @@ from databricks.labs.dqx.config import InputConfig, OutputConfig from databricks.labs.blueprint.installation import Installation, NotInstalled from databricks.sdk import WorkspaceClient +from databricks.sdk.errors import ResourceDoesNotExist logger = logging.getLogger(__name__) @@ -345,7 +346,7 @@ def get_custom_installation(ws: WorkspaceClient, product_name: str, install_fold """ try: ws.workspace.get_status(install_folder) - except NotInstalled: + except (NotInstalled, ResourceDoesNotExist): ws.workspace.mkdirs(install_folder) return Installation(ws, product_name, install_folder=install_folder) From 4054d40dd6f4cec0a0f9fb74c1edaa9bf3119e5f Mon Sep 17 00:00:00 2001 From: Greg Hansen Date: Tue, 16 Sep 2025 17:10:32 -0400 Subject: [PATCH 07/31] Fix tests --- src/databricks/labs/dqx/config.py | 3 +++ src/databricks/labs/dqx/utils.py | 4 ++-- tests/integration/test_installation.py | 5 +++-- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/src/databricks/labs/dqx/config.py b/src/databricks/labs/dqx/config.py index afacb08d1..836abcb31 100644 --- a/src/databricks/labs/dqx/config.py +++ b/src/databricks/labs/dqx/config.py @@ -226,4 +226,7 @@ class InstallationChecksStorageConfig( run_config_name: str = "default" # to retrieve run config product_name: str = "dqx" assume_user: bool = True + # DQX will be installed in a default directory if no custom folder is provided: + # - "/Applications/dqx" if `DQX_FORCE_INSTALL=global` during installation + # - "/Users//.dqx" otherwise install_folder: str | None = None diff --git a/src/databricks/labs/dqx/utils.py b/src/databricks/labs/dqx/utils.py index e22fe4fea..54fb4aa02 100644 --- a/src/databricks/labs/dqx/utils.py +++ b/src/databricks/labs/dqx/utils.py @@ -16,7 +16,7 @@ from databricks.labs.dqx.config import InputConfig, OutputConfig from databricks.labs.blueprint.installation import Installation, NotInstalled from databricks.sdk import WorkspaceClient -from databricks.sdk.errors import ResourceDoesNotExist +from databricks.sdk.errors import ResourceDoesNotExist, NotFound logger = logging.getLogger(__name__) @@ -359,7 +359,7 @@ def get_custom_installation(ws: WorkspaceClient, product_name: str, install_fold """ try: ws.workspace.get_status(install_folder) - except (NotInstalled, ResourceDoesNotExist): + except (NotInstalled, ResourceDoesNotExist, NotFound): ws.workspace.mkdirs(install_folder) return Installation(ws, product_name, install_folder=install_folder) diff --git a/tests/integration/test_installation.py b/tests/integration/test_installation.py index 9dc0641b6..4f36f5da4 100644 --- a/tests/integration/test_installation.py +++ b/tests/integration/test_installation.py @@ -15,6 +15,7 @@ from databricks.labs.blueprint.tui import MockPrompts from databricks.labs.blueprint.wheels import ProductInfo from databricks.labs.dqx.config import WorkspaceConfig, RunConfig, OutputConfig, ProfilerConfig, InputConfig +from databricks.labs.dqx.utils import get_custom_installation from databricks.sdk.errors import NotFound from databricks.sdk.service.jobs import CreateResponse from databricks.sdk import WorkspaceClient @@ -355,7 +356,7 @@ def test_custom_folder_installation(ws, new_installation, make_directory): product_info = ProductInfo.for_testing(WorkspaceConfig) custom_folder = str(make_directory().absolute()) - custom_installation = Installation(ws, product_info.product_name(), install_folder=custom_folder) + custom_installation = get_custom_installation(ws, product_info.product_name(), custom_folder) installation = new_installation( product_info=product_info, installation=custom_installation, @@ -369,7 +370,7 @@ def test_custom_folder_installation_with_environment_variable(ws, new_installati product_info = ProductInfo.for_testing(WorkspaceConfig) custom_folder = str(make_directory().absolute()) - custom_installation = Installation(ws, product_info.product_name(), install_folder=custom_folder) + custom_installation = get_custom_installation(ws, product_info.product_name(), custom_folder) installation = new_installation( product_info=product_info, installation=custom_installation, From ddd2d6fac9bb4d66675d23d5b090075229d5332d Mon Sep 17 00:00:00 2001 From: Greg Hansen Date: Tue, 16 Sep 2025 17:19:12 -0400 Subject: [PATCH 08/31] Format --- src/databricks/labs/dqx/utils.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/databricks/labs/dqx/utils.py b/src/databricks/labs/dqx/utils.py index 54fb4aa02..bced31caa 100644 --- a/src/databricks/labs/dqx/utils.py +++ b/src/databricks/labs/dqx/utils.py @@ -14,9 +14,9 @@ ConnectColumn = None # type: ignore from databricks.labs.dqx.config import InputConfig, OutputConfig -from databricks.labs.blueprint.installation import Installation, NotInstalled +from databricks.labs.blueprint.installation import Installation from databricks.sdk import WorkspaceClient -from databricks.sdk.errors import ResourceDoesNotExist, NotFound +from databricks.sdk.errors import NotFound logger = logging.getLogger(__name__) @@ -359,7 +359,7 @@ def get_custom_installation(ws: WorkspaceClient, product_name: str, install_fold """ try: ws.workspace.get_status(install_folder) - except (NotInstalled, ResourceDoesNotExist, NotFound): + except NotFound: ws.workspace.mkdirs(install_folder) return Installation(ws, product_name, install_folder=install_folder) From bc2d82ae6b145f9dc3fc059acfadec1a7693d9e6 Mon Sep 17 00:00:00 2001 From: Marcin Wojtyczka Date: Thu, 18 Sep 2025 11:02:12 +0200 Subject: [PATCH 09/31] refactor --- docs/dqx/docs/installation.mdx | 8 ++++---- src/databricks/labs/dqx/config.py | 7 ++++--- src/databricks/labs/dqx/installer/config_provider.py | 6 +++++- 3 files changed, 13 insertions(+), 8 deletions(-) diff --git a/docs/dqx/docs/installation.mdx b/docs/dqx/docs/installation.mdx index 97e851833..a9ff20d41 100644 --- a/docs/dqx/docs/installation.mdx +++ b/docs/dqx/docs/installation.mdx @@ -108,7 +108,7 @@ If you provide a custom path during installation (e.g., `/Shared/dqx-team` or `/ **Environment Variable Override:** You can also force global or user installation using the 'DQX_FORCE_INSTALL' environment variable: * `DQX_FORCE_INSTALL=global databricks labs install dqx`: forces installation to `/Applications/dqx` - * `DQX_FORCE_INSTALL=user databricks labs install dqx`: forces installation to `/Users//.dqx` + * `DQX_FORCE_INSTALL=user databricks labs install dqx`: forces installation to `/Users//.dqx` (default behaviour) If a custom folder is provided during installation, the installation folder will take precedence over any environment variables (e.g. `DQX_FORCE_INSTALL`). @@ -117,10 +117,10 @@ If a custom folder is provided during installation, the installation folder will #### Configuration file DQX configuration file can contain multiple run configurations for different pipelines or projects, each defining specific input, output and quarantine locations, etc. -The configuration file is created in the installation directory: -- Custom folder: `/config.yml` +The configuration file is created in the installation directory depending on installation options (see above): - User home (default): `/Users//.dqx/config.yml` -- Global: `/Applications/dqx/config.yml` +- Global: `/Applications/dqx/config.yml` (when DQX_FORCE_INSTALL=global) +- Custom folder: `/config.yml` The "default" run configuration is created during the installation. When DQX is upgraded, the configuration is preserved. The configuration can be updated / extended manually by the user after the installation. Each run config defines configuration for one specific input and output location. diff --git a/src/databricks/labs/dqx/config.py b/src/databricks/labs/dqx/config.py index 836abcb31..fd0be4f8d 100644 --- a/src/databricks/labs/dqx/config.py +++ b/src/databricks/labs/dqx/config.py @@ -219,7 +219,8 @@ class InstallationChecksStorageConfig( run_config_name: The name of the run configuration to use for checks (default is 'default'). product_name: The product name for retrieving checks from the installation (default is 'dqx'). assume_user: Whether to assume the user is the owner of the checks (default is True). - install_folder: The installation folder where DQX is installed + install_folder: The installation folder where DQX is installed. + Default to /Users//.dqx if not provided. """ location: str = "installation" # retrieved from the installation config @@ -227,6 +228,6 @@ class InstallationChecksStorageConfig( product_name: str = "dqx" assume_user: bool = True # DQX will be installed in a default directory if no custom folder is provided: - # - "/Applications/dqx" if `DQX_FORCE_INSTALL=global` during installation - # - "/Users//.dqx" otherwise + # * Default: "/Users//.dqx" + # * if `DQX_FORCE_INSTALL=global`: "/Applications/dqx" install_folder: str | None = None diff --git a/src/databricks/labs/dqx/installer/config_provider.py b/src/databricks/labs/dqx/installer/config_provider.py index 575ee3a05..fc188d7b9 100644 --- a/src/databricks/labs/dqx/installer/config_provider.py +++ b/src/databricks/labs/dqx/installer/config_provider.py @@ -1,3 +1,4 @@ +import os import json import logging from databricks.labs.blueprint.tui import Prompts @@ -27,7 +28,10 @@ def prompt_new_installation(self, install_folder: str | None = None) -> Workspac if install_folder: logger.info(f"DQX will be installed in folder '{install_folder}'") else: - logger.info("DQX will be installed in the default location (user home or global based on environment)") + install_path = ( + "/Applications/dqx" if os.getenv("DQX_FORCE_INSTALL") == "global" else "/Users//.dqx" + ) + logger.info(f"DQX will be installed in the default location: '{install_path}'") log_level = self._prompts.question("Log level", default="INFO").upper() is_streaming = self._prompts.confirm("Should the input data be read using streaming?") From dc4302512352d3c04bdb1286830fd6ece3d7379c Mon Sep 17 00:00:00 2001 From: Marcin Wojtyczka Date: Thu, 18 Sep 2025 11:24:05 +0200 Subject: [PATCH 10/31] refactor --- src/databricks/labs/dqx/config_loader.py | 25 ++++++++++++++++++-- src/databricks/labs/dqx/installer/install.py | 15 ++++++------ src/databricks/labs/dqx/utils.py | 24 ------------------- tests/integration/test_installation.py | 6 ++--- 4 files changed, 34 insertions(+), 36 deletions(-) diff --git a/src/databricks/labs/dqx/config_loader.py b/src/databricks/labs/dqx/config_loader.py index 21bd37ee8..4d294b8f9 100644 --- a/src/databricks/labs/dqx/config_loader.py +++ b/src/databricks/labs/dqx/config_loader.py @@ -1,8 +1,8 @@ from databricks.labs.blueprint.installation import Installation from databricks.sdk import WorkspaceClient +from databricks.sdk.errors import NotFound from databricks.labs.dqx.config import RunConfig, WorkspaceConfig -from databricks.labs.dqx.utils import get_custom_installation class RunConfigLoader: @@ -43,7 +43,7 @@ def get_installation(self, assume_user: bool, product_name: str, install_folder: """ if install_folder: - installation = get_custom_installation(self.ws, product_name, install_folder) + installation = self.get_custom_installation(self.ws, product_name, install_folder) elif assume_user: installation = Installation.assume_user_home(self.ws, product_name) else: @@ -52,6 +52,27 @@ def get_installation(self, assume_user: bool, product_name: str, install_folder: installation.current(self.ws, product_name, assume_user=assume_user) return installation + @staticmethod + def get_custom_installation(ws: WorkspaceClient, product_name: str, install_folder: str) -> Installation: + """ + Creates an Installation instance for a custom folder, similar to assume_user_home and assume_global. + This ensures the custom folder is created in the workspace when the installation is accessed. + + Args: + ws: Databricks SDK `WorkspaceClient` + product_name: The product name + install_folder: The custom installation folder path + + Returns: + An Installation instance for the custom folder + """ + try: + ws.workspace.get_status(install_folder) + except NotFound: + ws.workspace.mkdirs(install_folder) + + return Installation(ws, product_name, install_folder=install_folder) + @staticmethod def _load_run_config(installation: Installation, run_config_name: str | None) -> RunConfig: """ diff --git a/src/databricks/labs/dqx/installer/install.py b/src/databricks/labs/dqx/installer/install.py index e85691616..6888f3b7f 100644 --- a/src/databricks/labs/dqx/installer/install.py +++ b/src/databricks/labs/dqx/installer/install.py @@ -22,6 +22,7 @@ PermissionDenied, ) +from databricks.labs.dqx.config_loader import RunConfigLoader from databricks.labs.dqx.installer.config_provider import ConfigProvider from databricks.labs.dqx.installer.dashboard_installer import DashboardInstaller from databricks.labs.dqx.installer.version_checker import VersionChecker @@ -32,7 +33,6 @@ from databricks.labs.dqx.__about__ import __version__ from databricks.labs.dqx.config import WorkspaceConfig from databricks.labs.dqx.contexts.workspace_context import WorkspaceContext -from databricks.labs.dqx.utils import get_custom_installation logger = logging.getLogger(__name__) @@ -82,13 +82,14 @@ def installation(self): Raises: NotFound: If the installation is not found. """ + if self._install_folder: + return RunConfigLoader.get_custom_installation( + self.workspace_client, self.product_info.product_name(), self._install_folder + ) + try: return self.product_info.current_installation(self.workspace_client) except NotFound: - if self._install_folder: - return get_custom_installation( - self.workspace_client, self.product_info.product_name(), self._install_folder - ) if self._force_install == "global": return Installation.assume_global(self.workspace_client, self.product_info.product_name()) return Installation.assume_user_home(self.workspace_client, self.product_info.product_name()) @@ -378,9 +379,9 @@ def _create_dashboard(self) -> None: "Enter a workspace path for DQX installation (leave empty for default behavior)", default="", valid_regex=r"^(/.*)?$", - ) + ).strip() - custom_install_folder = custom_folder.strip() if custom_folder.strip() else None + custom_install_folder = custom_folder if custom_folder else None workspace_installer = WorkspaceInstaller( WorkspaceClient(product="dqx", product_version=__version__), install_folder=custom_install_folder diff --git a/src/databricks/labs/dqx/utils.py b/src/databricks/labs/dqx/utils.py index bced31caa..fd81e54c9 100644 --- a/src/databricks/labs/dqx/utils.py +++ b/src/databricks/labs/dqx/utils.py @@ -14,9 +14,6 @@ ConnectColumn = None # type: ignore from databricks.labs.dqx.config import InputConfig, OutputConfig -from databricks.labs.blueprint.installation import Installation -from databricks.sdk import WorkspaceClient -from databricks.sdk.errors import NotFound logger = logging.getLogger(__name__) @@ -342,24 +339,3 @@ def safe_json_load(value: str): return json.loads(value) # load as json if possible except json.JSONDecodeError: return value - - -def get_custom_installation(ws: WorkspaceClient, product_name: str, install_folder: str) -> Installation: - """ - Creates an Installation instance for a custom folder, similar to assume_user_home and assume_global. - This ensures the custom folder is created in the workspace when the installation is accessed. - - Args: - ws: Databricks SDK `WorkspaceClient` - product_name: The product name - install_folder: The custom installation folder path - - Returns: - An Installation instance for the custom folder - """ - try: - ws.workspace.get_status(install_folder) - except NotFound: - ws.workspace.mkdirs(install_folder) - - return Installation(ws, product_name, install_folder=install_folder) diff --git a/tests/integration/test_installation.py b/tests/integration/test_installation.py index 4f36f5da4..61882a375 100644 --- a/tests/integration/test_installation.py +++ b/tests/integration/test_installation.py @@ -3,6 +3,7 @@ from unittest.mock import patch, create_autospec import pytest +from databricks.labs.dqx.config_loader import RunConfigLoader from databricks.labs.dqx.installer.install import WorkspaceInstaller from tests.integration.conftest import contains_expected_workflows import databricks @@ -15,7 +16,6 @@ from databricks.labs.blueprint.tui import MockPrompts from databricks.labs.blueprint.wheels import ProductInfo from databricks.labs.dqx.config import WorkspaceConfig, RunConfig, OutputConfig, ProfilerConfig, InputConfig -from databricks.labs.dqx.utils import get_custom_installation from databricks.sdk.errors import NotFound from databricks.sdk.service.jobs import CreateResponse from databricks.sdk import WorkspaceClient @@ -356,7 +356,7 @@ def test_custom_folder_installation(ws, new_installation, make_directory): product_info = ProductInfo.for_testing(WorkspaceConfig) custom_folder = str(make_directory().absolute()) - custom_installation = get_custom_installation(ws, product_info.product_name(), custom_folder) + custom_installation = RunConfigLoader.get_custom_installation(ws, product_info.product_name(), custom_folder) installation = new_installation( product_info=product_info, installation=custom_installation, @@ -370,7 +370,7 @@ def test_custom_folder_installation_with_environment_variable(ws, new_installati product_info = ProductInfo.for_testing(WorkspaceConfig) custom_folder = str(make_directory().absolute()) - custom_installation = get_custom_installation(ws, product_info.product_name(), custom_folder) + custom_installation = RunConfigLoader.get_custom_installation(ws, product_info.product_name(), custom_folder) installation = new_installation( product_info=product_info, installation=custom_installation, From 379602ce2b0fb97ed58b58454f9fc45b9df1a9c0 Mon Sep 17 00:00:00 2001 From: Marcin Wojtyczka Date: Thu, 18 Sep 2025 13:05:59 +0200 Subject: [PATCH 11/31] refactor --- src/databricks/labs/dqx/config_loader.py | 4 ++- src/databricks/labs/dqx/installer/install.py | 6 ++-- tests/conftest.py | 21 ++++++++++-- tests/integration/test_config.py | 14 ++++++++ tests/integration/test_installation.py | 34 ++++---------------- 5 files changed, 45 insertions(+), 34 deletions(-) diff --git a/src/databricks/labs/dqx/config_loader.py b/src/databricks/labs/dqx/config_loader.py index 4d294b8f9..dc8e446f0 100644 --- a/src/databricks/labs/dqx/config_loader.py +++ b/src/databricks/labs/dqx/config_loader.py @@ -44,7 +44,9 @@ def get_installation(self, assume_user: bool, product_name: str, install_folder: if install_folder: installation = self.get_custom_installation(self.ws, product_name, install_folder) - elif assume_user: + return installation + + if assume_user: installation = Installation.assume_user_home(self.ws, product_name) else: installation = Installation.assume_global(self.ws, product_name) diff --git a/src/databricks/labs/dqx/installer/install.py b/src/databricks/labs/dqx/installer/install.py index 6888f3b7f..9c08e8e84 100644 --- a/src/databricks/labs/dqx/installer/install.py +++ b/src/databricks/labs/dqx/installer/install.py @@ -13,7 +13,7 @@ from databricks.labs.blueprint.parallel import ManyError, Threads from databricks.labs.blueprint.tui import Prompts from databricks.labs.blueprint.upgrades import Upgrades -from databricks.labs.blueprint.wheels import ProductInfo +from databricks.labs.blueprint.wheels import ProductInfo, WheelsV2 from databricks.sdk import WorkspaceClient from databricks.sdk.core import with_user_agent_extra from databricks.sdk.errors import ( @@ -271,7 +271,7 @@ def __init__( self._ws = ws self._prompts = prompts self._product_info = product_info - self._wheels = product_info.wheels(ws) + self._wheels = WheelsV2(self._installation, product_info) @classmethod def current(cls, ws: WorkspaceClient): @@ -290,7 +290,7 @@ def current(cls, ws: WorkspaceClient): config = installation.load(WorkspaceConfig) run_config_name = config.get_run_config().name prompts = Prompts() - wheels = product_info.wheels(ws) + wheels = WheelsV2(installation, product_info) tasks = WorkflowsRunner.all(config).tasks() workflow_installer = WorkflowDeployment( config, run_config_name, installation, install_state, ws, wheels, product_info, tasks diff --git a/tests/conftest.py b/tests/conftest.py index 7070e709d..5766c0846 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -6,7 +6,7 @@ import pytest from databricks.labs.blueprint.installation import Installation, MockInstallation from databricks.labs.blueprint.tui import MockPrompts -from databricks.labs.blueprint.wheels import ProductInfo +from databricks.labs.blueprint.wheels import ProductInfo, WheelsV2 from databricks.labs.dqx.__about__ import __version__ from databricks.labs.dqx.config import WorkspaceConfig, RunConfig from databricks.labs.dqx.contexts.workflow_context import WorkflowContext @@ -78,14 +78,16 @@ def __init__( ws: WorkspaceClient, checks_location, serverless_clusters: bool = True, + install_folder: str | None = None, ): super().__init__(env_or_skip_fixture, ws) self.checks_location = checks_location self.serverless_clusters = serverless_clusters + self.install_folder = install_folder @cached_property def installation(self): - return Installation(self.workspace_client, self.product_info.product_name()) + return Installation(self.workspace_client, self.product_info.product_name(), install_folder=self.install_folder) @cached_property def environ(self) -> dict[str, str]: @@ -96,6 +98,7 @@ def workspace_installer(self): return WorkspaceInstaller( self.workspace_client, self.environ, + self.install_folder, ).replace(prompts=self.prompts, installation=self.installation, product_info=self.product_info) @cached_property @@ -130,7 +133,7 @@ def workflows_deployment(self) -> WorkflowDeployment: self.installation, self.install_state, self.workspace_client, - self.product_info.wheels(self.workspace_client), + WheelsV2(self.installation, self.product_info), self.product_info, self.tasks, ) @@ -188,6 +191,18 @@ def serverless_installation_ctx( ctx.installation_service.uninstall() +@pytest.fixture +def installation_ctx_custom_install_folder( + ws: WorkspaceClient, make_directory, env_or_skip: Callable[[str], str], checks_location="checks.yml" +) -> Generator[MockInstallationContext, None, None]: + custom_folder = str(make_directory().absolute()) + ctx = MockInstallationContext( + env_or_skip, ws, checks_location, serverless_clusters=False, install_folder=custom_folder + ) + yield ctx.replace(workspace_client=ws) + ctx.installation_service.uninstall() + + @pytest.fixture def checks_yaml_content(): return """- criticality: error diff --git a/tests/integration/test_config.py b/tests/integration/test_config.py index 9d5daffa6..54558d6a1 100644 --- a/tests/integration/test_config.py +++ b/tests/integration/test_config.py @@ -1,6 +1,9 @@ from unittest.mock import patch from databricks.labs.blueprint.installation import Installation +from databricks.labs.blueprint.wheels import ProductInfo + +from databricks.labs.dqx.config import WorkspaceConfig from databricks.labs.dqx.config_loader import RunConfigLoader @@ -29,3 +32,14 @@ def test_load_run_config_from_global_installation(ws, installation_ctx): ) assert run_config == expected_run_config + + +def test_get_custom_installation(ws, make_directory): + product_info = ProductInfo.for_testing(WorkspaceConfig) + custom_folder = str(make_directory().absolute()) + + custom_installation = RunConfigLoader.get_custom_installation(ws, product_info.product_name(), custom_folder) + custom_installation.install_folder() + + assert custom_installation.install_folder() == custom_folder + assert ws.workspace.get_status(custom_folder) diff --git a/tests/integration/test_installation.py b/tests/integration/test_installation.py index 61882a375..f6226041d 100644 --- a/tests/integration/test_installation.py +++ b/tests/integration/test_installation.py @@ -3,7 +3,6 @@ from unittest.mock import patch, create_autospec import pytest -from databricks.labs.dqx.config_loader import RunConfigLoader from databricks.labs.dqx.installer.install import WorkspaceInstaller from tests.integration.conftest import contains_expected_workflows import databricks @@ -352,33 +351,14 @@ def test_workflows_deployment_creates_jobs_with_remove_after_tag(): wheels.assert_not_called() -def test_custom_folder_installation(ws, new_installation, make_directory): - product_info = ProductInfo.for_testing(WorkspaceConfig) - custom_folder = str(make_directory().absolute()) - - custom_installation = RunConfigLoader.get_custom_installation(ws, product_info.product_name(), custom_folder) - installation = new_installation( - product_info=product_info, - installation=custom_installation, - ) - - assert installation.install_folder() == custom_folder - assert ws.workspace.get_status(custom_folder) - - -def test_custom_folder_installation_with_environment_variable(ws, new_installation, make_directory): - product_info = ProductInfo.for_testing(WorkspaceConfig) - custom_folder = str(make_directory().absolute()) - - custom_installation = RunConfigLoader.get_custom_installation(ws, product_info.product_name(), custom_folder) - installation = new_installation( - product_info=product_info, - installation=custom_installation, - environ={'DQX_FORCE_INSTALL': 'global'}, # environment variable should not override the install folder - ) +def test_custom_folder_installation(ws, installation_ctx_custom_install_folder): + installation_ctx_custom_install_folder.installation_service.run() + workflows = installation_ctx_custom_install_folder.deployed_workflows.latest_job_status() + expected_workflows_state = [{'workflow': 'profiler', 'state': 'UNKNOWN', 'started': ''}] - assert installation.install_folder() == custom_folder - assert ws.workspace.get_status(custom_folder) + assert ws.workspace.get_status(installation_ctx_custom_install_folder.installation_service.install_folder) + for state in expected_workflows_state: + assert contains_expected_workflows(workflows, state) def test_my_username(): From 04fc28710dff1dbd27f297d15731e4e383485f7d Mon Sep 17 00:00:00 2001 From: Marcin Wojtyczka Date: Thu, 18 Sep 2025 14:45:07 +0200 Subject: [PATCH 12/31] added integration tests enable e2e tests on nightly --- .github/workflows/nightly.yml | 2 - tests/integration/test_installation.py | 180 +++++++++++++++++++++++-- 2 files changed, 169 insertions(+), 13 deletions(-) diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 835ca416e..23b06c7e0 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -97,7 +97,6 @@ jobs: DATABRICKS_SERVERLESS_COMPUTE_ID: ${{ env.DATABRICKS_SERVERLESS_COMPUTE_ID }} e2e: - if: github.event_name == 'pull_request' && !github.event.pull_request.draft && !github.event.pull_request.head.repo.fork environment: tool runs-on: larger steps: @@ -147,7 +146,6 @@ jobs: ARM_TENANT_ID: ${{ secrets.ARM_TENANT_ID }} e2e_serverless: - if: github.event_name == 'pull_request' && !github.event.pull_request.draft && !github.event.pull_request.head.repo.fork environment: tool runs-on: larger env: diff --git a/tests/integration/test_installation.py b/tests/integration/test_installation.py index f6226041d..b88508a46 100644 --- a/tests/integration/test_installation.py +++ b/tests/integration/test_installation.py @@ -3,6 +3,7 @@ from unittest.mock import patch, create_autospec import pytest +from databricks.labs.dqx.config_loader import RunConfigLoader from databricks.labs.dqx.installer.install import WorkspaceInstaller from tests.integration.conftest import contains_expected_workflows import databricks @@ -33,6 +34,7 @@ def factory( product_info: ProductInfo | None = None, environ: dict[str, str] | None = None, extend_prompts: dict[str, str] | None = None, + install_folder: str | None = None, ): logger.debug("Creating new installation...") if not product_info: @@ -53,12 +55,14 @@ def factory( if not installation: installation = Installation(ws, product_info.product_name()) - installer = WorkspaceInstaller(ws, environ).replace( + installer = WorkspaceInstaller(ws, environ, install_folder=install_folder).replace( installation=installation, product_info=product_info, prompts=prompts, ) workspace_config = installer.configure() + if not install_folder: + installation = product_info.current_installation(ws) installation.save(workspace_config) cleanup.append(installation) return installation @@ -89,12 +93,26 @@ def test_fresh_user_config_installation(ws, installation_ctx): ) +def test_fresh_custom_folder_config_installation(ws, installation_ctx_custom_install_folder): + installation_ctx_custom_install_folder.installation.save(installation_ctx_custom_install_folder.config) + assert ( + installation_ctx_custom_install_folder.installation_service.install_folder + != f"/Users/{ws.current_user.me().user_name}/.{installation_ctx_custom_install_folder.product_info.product_name()}" + ) + + def test_complete_installation(ws, installation_ctx): installation_ctx.workspace_installer.run(installation_ctx.config) assert installation_ctx.workspace_installer.installation assert installation_ctx.deployed_workflows.latest_job_status() +def test_complete_installation_with_custom_folder(ws, installation_ctx_custom_install_folder): + installation_ctx_custom_install_folder.workspace_installer.run(installation_ctx_custom_install_folder.config) + assert installation_ctx_custom_install_folder.workspace_installer.installation + assert installation_ctx_custom_install_folder.deployed_workflows.latest_job_status() + + def test_installation(ws, installation_ctx): installation_ctx.installation_service.run() workflows = installation_ctx.deployed_workflows.latest_job_status() @@ -105,6 +123,16 @@ def test_installation(ws, installation_ctx): assert contains_expected_workflows(workflows, state) +def test_installation_with_custom_folder(ws, installation_ctx_custom_install_folder): + installation_ctx_custom_install_folder.installation_service.run() + workflows = installation_ctx_custom_install_folder.deployed_workflows.latest_job_status() + expected_workflows_state = [{'workflow': 'profiler', 'state': 'UNKNOWN', 'started': ''}] + + assert ws.workspace.get_status(installation_ctx_custom_install_folder.installation_service.install_folder) + for state in expected_workflows_state: + assert contains_expected_workflows(workflows, state) + + def test_dashboard_state_installation(ws, installation_ctx): installation_ctx.installation_service.run() dashboard_id = list(installation_ctx.install_state.dashboards.values())[0] @@ -112,6 +140,13 @@ def test_dashboard_state_installation(ws, installation_ctx): assert dashboard_id is not None +def test_dashboard_state_installation_with_custom_folder(ws, installation_ctx_custom_install_folder): + installation_ctx_custom_install_folder.installation_service.run() + dashboard_id = list(installation_ctx_custom_install_folder.install_state.dashboards.values())[0] + + assert dashboard_id is not None + + def test_dashboard_workspace_installation(ws, installation_ctx): installation_ctx.installation_service.run() dashboard_id = list(installation_ctx.install_state.dashboards.values())[0] @@ -120,6 +155,14 @@ def test_dashboard_workspace_installation(ws, installation_ctx): assert dashboard.lifecycle_state == LifecycleState.ACTIVE +def test_dashboard_workspace_installation_with_custom_folder(ws, installation_ctx_custom_install_folder): + installation_ctx_custom_install_folder.installation_service.run() + dashboard_id = list(installation_ctx_custom_install_folder.install_state.dashboards.values())[0] + dashboard = ws.lakeview.get(dashboard_id) + + assert dashboard.lifecycle_state == LifecycleState.ACTIVE + + def test_dashboard_repeated_workspace_installation(ws, installation_ctx): installation_ctx.installation_service.run() installation_ctx.installation_service.run() @@ -129,6 +172,15 @@ def test_dashboard_repeated_workspace_installation(ws, installation_ctx): assert dashboard.lifecycle_state == LifecycleState.ACTIVE +def test_dashboard_repeated_workspace_installation_with_custom_folder(ws, installation_ctx_custom_install_folder): + installation_ctx_custom_install_folder.installation_service.run() + installation_ctx_custom_install_folder.installation_service.run() + dashboard_id = list(installation_ctx_custom_install_folder.install_state.dashboards.values())[0] + dashboard = ws.lakeview.get(dashboard_id) + + assert dashboard.lifecycle_state == LifecycleState.ACTIVE + + def test_installation_when_dashboard_is_trashed(ws, installation_ctx): """A dashboard might be trashed (manually), the upgrade should handle this.""" installation_ctx.installation_service.run() @@ -141,6 +193,18 @@ def test_installation_when_dashboard_is_trashed(ws, installation_ctx): assert True, "Installation succeeded when dashboard was trashed" +def test_installation_with_custom_folder_when_dashboard_is_trashed(ws, installation_ctx_custom_install_folder): + """A dashboard might be trashed (manually), the upgrade should handle this.""" + installation_ctx_custom_install_folder.installation_service.run() + dashboard_id = list(installation_ctx_custom_install_folder.install_state.dashboards.values())[0] + ws.lakeview.trash(dashboard_id) + try: + installation_ctx_custom_install_folder.installation_service.run() + except NotFound: + assert False, "Installation failed when dashboard was trashed" + assert True, "Installation succeeded when dashboard was trashed" + + def test_installation_when_dashboard_state_missing(ws, installation_ctx): installation_ctx.installation_service.run() state_file = installation_ctx.install_state.install_folder() + "/" + RawState.__file__ @@ -152,6 +216,17 @@ def test_installation_when_dashboard_state_missing(ws, installation_ctx): assert dashboard.lifecycle_state == LifecycleState.ACTIVE +def test_installation_with_custom_folder_when_dashboard_state_missing(ws, installation_ctx_custom_install_folder): + installation_ctx_custom_install_folder.installation_service.run() + state_file = installation_ctx_custom_install_folder.install_state.install_folder() + "/" + RawState.__file__ + ws.workspace.delete(state_file) + installation_ctx_custom_install_folder.installation_service.run() # check that dashboard can be overwritten + dashboard_id = list(installation_ctx_custom_install_folder.install_state.dashboards.values())[0] + dashboard = ws.lakeview.get(dashboard_id) + + assert dashboard.lifecycle_state == LifecycleState.ACTIVE + + def test_uninstallation(ws, installation_ctx): installation_ctx.installation_service.run() job_id = list(installation_ctx.install_state.jobs.values())[0] @@ -165,6 +240,19 @@ def test_uninstallation(ws, installation_ctx): ws.dashboards.get(dashboard_id) +def test_uninstallation_with_custom_folder(ws, installation_ctx_custom_install_folder): + installation_ctx_custom_install_folder.installation_service.run() + job_id = list(installation_ctx_custom_install_folder.install_state.jobs.values())[0] + dashboard_id = list(installation_ctx_custom_install_folder.install_state.dashboards.values())[0] + installation_ctx_custom_install_folder.installation_service.uninstall() + with pytest.raises(NotFound): + ws.workspace.get_status(installation_ctx_custom_install_folder.installation_service.install_folder) + with pytest.raises(NotFound): + ws.jobs.get(job_id) + with pytest.raises(NotFound): + ws.dashboards.get(dashboard_id) + + def test_uninstallation_dashboard_does_not_exist_anymore(ws, installation_ctx): installation_ctx.installation_service.run() dashboard_id = list(installation_ctx.install_state.dashboards.values())[0] @@ -172,6 +260,13 @@ def test_uninstallation_dashboard_does_not_exist_anymore(ws, installation_ctx): installation_ctx.installation_service.uninstall() +def test_uninstallation_with_custom_folder_dashboard_does_not_exist_anymore(ws, installation_ctx_custom_install_folder): + installation_ctx_custom_install_folder.installation_service.run() + dashboard_id = list(installation_ctx_custom_install_folder.install_state.dashboards.values())[0] + ws.lakeview.trash(dashboard_id) + installation_ctx_custom_install_folder.installation_service.uninstall() + + def test_uninstallation_job_does_not_exist_anymore(ws, installation_ctx): installation_ctx.installation_service.run() job_id = list(installation_ctx.install_state.jobs.values())[0] @@ -179,6 +274,13 @@ def test_uninstallation_job_does_not_exist_anymore(ws, installation_ctx): installation_ctx.installation_service.uninstall() +def test_uninstallation_with_custom_folder_job_does_not_exist_anymore(ws, installation_ctx_custom_install_folder): + installation_ctx_custom_install_folder.installation_service.run() + job_id = list(installation_ctx_custom_install_folder.install_state.jobs.values())[0] + ws.jobs.delete(job_id) + installation_ctx_custom_install_folder.installation_service.uninstall() + + def test_global_installation_on_existing_global_install(ws, installation_ctx): product_name = installation_ctx.product_info.product_name() # patch the global installation to existing install_folder to avoid access permission issues in the workspace @@ -289,6 +391,61 @@ def test_global_installation_on_existing_user_install(ws, new_installation): ) +def test_custom_folder_installation_on_existing_user_installation(ws, make_directory, new_installation): + product_info = ProductInfo.for_testing(WorkspaceConfig) + + existing_user_installation = new_installation( + product_info=product_info, installation=Installation.assume_user_home(ws, product_info.product_name()) + ) + assert ( + existing_user_installation.install_folder() + == f"/Users/{ws.current_user.me().user_name}/.{product_info.product_name()}" + ) + + custom_folder = str(make_directory().absolute()) + custom_installation = RunConfigLoader.get_custom_installation(ws, product_info.product_name(), custom_folder) + installation = new_installation( + product_info=product_info, + installation=custom_installation, + install_folder=custom_folder, + ) + + assert installation.install_folder() == custom_folder + assert ws.workspace.get_status(custom_folder) + + +def test_custom_folder_installation_upgrade(ws, installation_ctx_custom_install_folder, new_installation): + product_info = ProductInfo.for_testing(WorkspaceConfig) + installation_ctx_custom_install_folder.installation.save(installation_ctx_custom_install_folder.config) + + custom_folder = installation_ctx_custom_install_folder.installation_service.install_folder + custom_installation = RunConfigLoader.get_custom_installation(ws, product_info.product_name(), custom_folder) + second_installation = new_installation( + product_info=product_info, + installation=custom_installation, + install_folder=custom_folder, + ) + + assert second_installation.install_folder() == custom_folder + assert ws.workspace.get_status(custom_folder) + + +def test_custom_folder_installation_with_environment_variable(ws, make_directory, new_installation): + product_info = ProductInfo.for_testing(WorkspaceConfig) + custom_folder = str(make_directory().absolute()) + + custom_installation = RunConfigLoader.get_custom_installation(ws, product_info.product_name(), custom_folder) + installation = new_installation( + product_info=product_info, + installation=custom_installation, + install_folder=custom_folder, + environ={'DQX_FORCE_INSTALL': 'global'}, # environment variable should not override the install folder + ) + + assert installation.install_folder() == custom_folder + assert ws.workspace.get_status(custom_folder) + + @skip("New tag version must be created in git") def test_compare_remote_local_install_versions(ws, installation_ctx): installation_ctx.installation_service.run() @@ -319,6 +476,17 @@ def test_installation_stores_install_state_keys(ws, installation_ctx): assert getattr(install_state, key), f"Installation state is empty: {key}" +def test_installation_with_custom_folder_stores_install_state_keys(ws, installation_ctx_custom_install_folder): + """The installation should store the keys in the installation state.""" + expected_keys = ["jobs", "dashboards"] + installation_ctx_custom_install_folder.installation_service.run() + # Refresh the installation state since the installation context uses `@cached_property` + install_state = InstallState.from_installation(installation_ctx_custom_install_folder.installation) + for key in expected_keys: + assert hasattr(install_state, key), f"Missing key in install state: {key}" + assert getattr(install_state, key), f"Installation state is empty: {key}" + + def side_effect_remove_after_in_tags_settings(**settings) -> CreateResponse: tags = settings.get("tags", {}) _ = tags["RemoveAfter"] # KeyError side effect @@ -351,16 +519,6 @@ def test_workflows_deployment_creates_jobs_with_remove_after_tag(): wheels.assert_not_called() -def test_custom_folder_installation(ws, installation_ctx_custom_install_folder): - installation_ctx_custom_install_folder.installation_service.run() - workflows = installation_ctx_custom_install_folder.deployed_workflows.latest_job_status() - expected_workflows_state = [{'workflow': 'profiler', 'state': 'UNKNOWN', 'started': ''}] - - assert ws.workspace.get_status(installation_ctx_custom_install_folder.installation_service.install_folder) - for state in expected_workflows_state: - assert contains_expected_workflows(workflows, state) - - def test_my_username(): """Test the _my_username property to cover both conditions.""" From 9e3da42764736ff7083cb4ba0119df6913b28f9b Mon Sep 17 00:00:00 2001 From: Marcin Wojtyczka Date: Thu, 18 Sep 2025 16:11:01 +0200 Subject: [PATCH 13/31] handle custom install folder with workflows --- docs/dqx/docs/reference/engine.mdx | 5 ++-- src/databricks/labs/dqx/checks_storage.py | 5 +++- .../labs/dqx/profiler/profiler_workflow.py | 1 + .../quality_checker_workflow.py | 1 + src/databricks/labs/dqx/workflows_runner.py | 5 ++-- .../test_load_checks_from_workspace_file.py | 17 ++++++++++++++ .../test_save_checks_to_workspace_file.py | 23 ++++++++++++++++++- 7 files changed, 51 insertions(+), 6 deletions(-) diff --git a/docs/dqx/docs/reference/engine.mdx b/docs/dqx/docs/reference/engine.mdx index 84ab7e441..8327b7f03 100644 --- a/docs/dqx/docs/reference/engine.mdx +++ b/docs/dqx/docs/reference/engine.mdx @@ -93,9 +93,10 @@ Supported storage backend configurations (implementations of `BaseChecksStorageC * `location`: Unity Catalog Volume file path (JSON or YAML) * `InstallationChecksStorageConfig` can be used to save or load checks from workspace installation, with fields: * `location` (optional): automatically set based on the `checks_location` field from the run configuration. + * `install_folder`: (optional) installation folder where DQX is installed, only required when custom installation folder is used * `run_config_name` (optional) - run configuration name to load (use "default" if not provided). - * `product_name`: name of the product/installation directory - * `assume_user`: if True, assume user installation + * `product_name`: (optional) name of the product/installation directory + * `assume_user`: (optional) if True, assume user installation For details on how to prepare reference DataFrames (`ref_dfs`) and custom check function mapping (`custom_check_functions`) refer to [Quality Checks Reference](/docs/reference/quality_checks). diff --git a/src/databricks/labs/dqx/checks_storage.py b/src/databricks/labs/dqx/checks_storage.py index c9dd98613..9607db022 100644 --- a/src/databricks/labs/dqx/checks_storage.py +++ b/src/databricks/labs/dqx/checks_storage.py @@ -262,7 +262,10 @@ def _get_storage_handler_and_config( self, config: InstallationChecksStorageConfig ) -> tuple[ChecksStorageHandler, InstallationChecksStorageConfig]: run_config = self._run_config_loader.load_run_config( - run_config_name=config.run_config_name, assume_user=config.assume_user, product_name=config.product_name + run_config_name=config.run_config_name, + assume_user=config.assume_user, + product_name=config.product_name, + install_folder=config.install_folder, ) installation = self._run_config_loader.get_installation( config.assume_user, config.product_name, config.install_folder diff --git a/src/databricks/labs/dqx/profiler/profiler_workflow.py b/src/databricks/labs/dqx/profiler/profiler_workflow.py index 6c178e1ab..d52d3cbde 100644 --- a/src/databricks/labs/dqx/profiler/profiler_workflow.py +++ b/src/databricks/labs/dqx/profiler/profiler_workflow.py @@ -35,6 +35,7 @@ def profile(self, ctx: WorkflowContext): run_config_name=run_config.name, assume_user=True, product_name=ctx.installation.product(), + install_folder=ctx.installation.install_folder(), ) ctx.profiler.save(checks, profile_summary_stats, storage_config, run_config.profiler_config.summary_stats_file) diff --git a/src/databricks/labs/dqx/quality_checker/quality_checker_workflow.py b/src/databricks/labs/dqx/quality_checker/quality_checker_workflow.py index fe51803ad..a652d4b9d 100644 --- a/src/databricks/labs/dqx/quality_checker/quality_checker_workflow.py +++ b/src/databricks/labs/dqx/quality_checker/quality_checker_workflow.py @@ -34,6 +34,7 @@ def apply_checks(self, ctx: WorkflowContext): location=run_config.checks_location, run_config_name=run_config.name, product_name=ctx.product_info.product_name(), + install_folder=ctx.installation.install_folder(), ) ) diff --git a/src/databricks/labs/dqx/workflows_runner.py b/src/databricks/labs/dqx/workflows_runner.py index 2d7add0dd..bfaea26ec 100644 --- a/src/databricks/labs/dqx/workflows_runner.py +++ b/src/databricks/labs/dqx/workflows_runner.py @@ -36,10 +36,11 @@ def __init__(self, workflows: list[Workflow]): self._tasks.append(with_workflow) @classmethod - def all(cls, config: WorkspaceConfig): + def all(cls, config: WorkspaceConfig) -> "WorkflowsRunner": """Return all workflows.""" profiler = ProfilerWorkflow( - spark_conf=config.profiler_spark_conf, override_clusters=config.profiler_override_clusters + spark_conf=config.profiler_spark_conf, + override_clusters=config.profiler_override_clusters, ) quality_checker = DataQualityWorkflow( spark_conf=config.quality_checker_spark_conf, diff --git a/tests/integration/test_load_checks_from_workspace_file.py b/tests/integration/test_load_checks_from_workspace_file.py index 22125243f..29ee069ef 100644 --- a/tests/integration/test_load_checks_from_workspace_file.py +++ b/tests/integration/test_load_checks_from_workspace_file.py @@ -96,6 +96,23 @@ def test_load_checks_from_user_installation(ws, installation_ctx, make_check_fil assert checks == expected_checks, "Checks were not loaded correctly" +def test_load_checks_from_custom_folder_installation( + ws, installation_ctx_custom_install_folder, make_check_file_as_yaml, expected_checks, spark +): + installation_ctx_custom_install_folder.installation.save(installation_ctx_custom_install_folder.config) + make_check_file_as_yaml(install_dir=installation_ctx_custom_install_folder.installation.install_folder()) + + config = InstallationChecksStorageConfig( + run_config_name="default", + assume_user=True, + product_name=installation_ctx_custom_install_folder.installation.product(), + install_folder=installation_ctx_custom_install_folder.installation.install_folder(), + ) + checks = DQEngine(ws, spark).load_checks(config=config) + + assert checks == expected_checks, "Checks were not loaded correctly" + + def test_load_checks_from_absolute_path(ws, installation_ctx, make_check_file_as_yaml, expected_checks, spark): checks_location = make_check_file_as_yaml() config = installation_ctx.config diff --git a/tests/integration/test_save_checks_to_workspace_file.py b/tests/integration/test_save_checks_to_workspace_file.py index 9883fd763..93ced82b6 100644 --- a/tests/integration/test_save_checks_to_workspace_file.py +++ b/tests/integration/test_save_checks_to_workspace_file.py @@ -111,7 +111,28 @@ def test_save_checks_when_global_installation_missing(ws, spark): DQEngine(ws, spark).save_checks(TEST_CHECKS, config=config) -def test_load_checks_when_user_installation_missing(ws, spark): +def test_save_checks_in_custom_folder_installation_in_yaml_file(ws, spark, installation_ctx_custom_install_folder): + installation_ctx_custom_install_folder.installation.save(installation_ctx_custom_install_folder.config) + product_name = installation_ctx_custom_install_folder.product_info.product_name() + + dq_engine = DQEngine(ws, spark) + config = InstallationChecksStorageConfig( + run_config_name="default", + assume_user=True, + product_name=product_name, + install_folder=installation_ctx_custom_install_folder.installation.install_folder(), + ) + dq_engine.save_checks(TEST_CHECKS, config=config) + + install_dir = installation_ctx_custom_install_folder.installation.install_folder() + checks_path = f"{install_dir}/{installation_ctx_custom_install_folder.config.get_run_config().checks_location}" + _verify_workspace_file_is_valid(ws, checks_path, file_format="yaml") + + checks = dq_engine.load_checks(config=config) + assert TEST_CHECKS == checks, "Checks were not saved correctly" + + +def test_save_checks_when_user_installation_missing(ws, spark): with pytest.raises(NotFound): config = InstallationChecksStorageConfig(run_config_name="default", assume_user=True) DQEngine(ws, spark).save_checks(TEST_CHECKS, config=config) From 2dcbdceb9a2540dfcc4f01da0bc9bd91b2e011e6 Mon Sep 17 00:00:00 2001 From: Marcin Wojtyczka Date: Thu, 18 Sep 2025 16:39:17 +0200 Subject: [PATCH 14/31] handle custom install folder with workflows --- docs/dqx/docs/installation.mdx | 4 ++++ src/databricks/labs/dqx/installer/config_provider.py | 12 +++++------- src/databricks/labs/dqx/installer/install.py | 8 ++++---- 3 files changed, 13 insertions(+), 11 deletions(-) diff --git a/docs/dqx/docs/installation.mdx b/docs/dqx/docs/installation.mdx index a9ff20d41..9d20dd345 100644 --- a/docs/dqx/docs/installation.mdx +++ b/docs/dqx/docs/installation.mdx @@ -105,6 +105,9 @@ DQX offers flexible installation options. By default, DQX is installed in the us **Custom Workspace Folder:** If you provide a custom path during installation (e.g., `/Shared/dqx-team` or `/Users/shared-user/dqx-project`), DQX will be installed there. +The custom folder installation is required when using [Group Assigned cluster](https://docs.databricks.com/aws/en/compute/group-access), +as the concept of a user home directory does not exist in this setup. + **Environment Variable Override:** You can also force global or user installation using the 'DQX_FORCE_INSTALL' environment variable: * `DQX_FORCE_INSTALL=global databricks labs install dqx`: forces installation to `/Applications/dqx` @@ -121,6 +124,7 @@ The configuration file is created in the installation directory depending on ins - User home (default): `/Users//.dqx/config.yml` - Global: `/Applications/dqx/config.yml` (when DQX_FORCE_INSTALL=global) - Custom folder: `/config.yml` + The "default" run configuration is created during the installation. When DQX is upgraded, the configuration is preserved. The configuration can be updated / extended manually by the user after the installation. Each run config defines configuration for one specific input and output location. diff --git a/src/databricks/labs/dqx/installer/config_provider.py b/src/databricks/labs/dqx/installer/config_provider.py index fc188d7b9..e55405525 100644 --- a/src/databricks/labs/dqx/installer/config_provider.py +++ b/src/databricks/labs/dqx/installer/config_provider.py @@ -6,32 +6,30 @@ from databricks.labs.dqx.config import WorkspaceConfig, RunConfig, InputConfig, OutputConfig, ProfilerConfig -logger = logging.getLogger(__name__) - - class ConfigProvider: """ Collects configuration from the user interactively. """ - def __init__(self, prompts: Prompts, warehouse_configurator: WarehouseInstaller): + def __init__(self, prompts: Prompts, warehouse_configurator: WarehouseInstaller, logger: logging.Logger): self._prompts = prompts self._warehouse_configurator = warehouse_configurator + self.logger = logger def prompt_new_installation(self, install_folder: str | None = None) -> WorkspaceConfig: - logger.info( + self.logger.info( "Please answer a couple of questions to provide default DQX run configuration. " "The configuration can also be updated manually after the installation." ) # Show installation folder information if install_folder: - logger.info(f"DQX will be installed in folder '{install_folder}'") + self.logger.info(f"DQX will be installed in folder '{install_folder}'") else: install_path = ( "/Applications/dqx" if os.getenv("DQX_FORCE_INSTALL") == "global" else "/Users//.dqx" ) - logger.info(f"DQX will be installed in the default location: '{install_path}'") + self.logger.info(f"DQX will be installed in the default location: '{install_path}'") log_level = self._prompts.question("Log level", default="INFO").upper() is_streaming = self._prompts.confirm("Should the input data be read using streaming?") diff --git a/src/databricks/labs/dqx/installer/install.py b/src/databricks/labs/dqx/installer/install.py index 9c08e8e84..fb4a2239e 100644 --- a/src/databricks/labs/dqx/installer/install.py +++ b/src/databricks/labs/dqx/installer/install.py @@ -211,7 +211,7 @@ def _is_testing(self): def _prompt_for_new_installation(self) -> WorkspaceConfig: configurator = WarehouseInstaller(self.workspace_client, self.prompts) - prompter = ConfigProvider(self.prompts, configurator) + prompter = ConfigProvider(self.prompts, configurator, logger) return prompter.prompt_new_installation(self._install_folder) def _confirm_force_install(self) -> bool: @@ -376,12 +376,12 @@ def _create_dashboard(self) -> None: installer_prompts = Prompts() custom_folder = installer_prompts.question( - "Enter a workspace path for DQX installation (leave empty for default behavior)", - default="", + "Enter a workspace path for DQX installation (leave empty to install in user's home or global directory)", + default="empty", valid_regex=r"^(/.*)?$", ).strip() - custom_install_folder = custom_folder if custom_folder else None + custom_install_folder = custom_folder if custom_folder and custom_folder != "empty" else None workspace_installer = WorkspaceInstaller( WorkspaceClient(product="dqx", product_version=__version__), install_folder=custom_install_folder From 288af80ec168cfd2ce08b755ad8a106ca71da967 Mon Sep 17 00:00:00 2001 From: Marcin Wojtyczka Date: Fri, 19 Sep 2025 06:36:10 +0200 Subject: [PATCH 15/31] added integration tests --- tests/integration/conftest.py | 32 +++++++++++++++++++ tests/integration/test_e2e_workflow.py | 23 +++++++++++++ tests/integration/test_profiler_workflow.py | 21 ++++++++++++ .../test_quality_checker_workflow.py | 11 +++++++ 4 files changed, 87 insertions(+) diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index d833aa199..33742c213 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -91,6 +91,38 @@ def delete(resource) -> None: yield from factory("workflows", lambda **kw: create(spark, **kw), delete) +@pytest.fixture +def setup_workflows_with_custom_install_folder( + ws, spark, installation_ctx_custom_install_folder, make_schema, make_table, make_random +): + """ + Set up the workflows with installation in the custom install folder. + """ + + def create(_spark, **kwargs): + installation_ctx_custom_install_folder.installation_service.run() + + quarantine = False + if "quarantine" in kwargs and kwargs["quarantine"]: + quarantine = True + + checks_location = None + if "checks" in kwargs and kwargs["checks"]: + checks_location = _setup_quality_checks(installation_ctx_custom_install_folder, _spark, ws) + + run_config = _setup_workflows_deps( + installation_ctx_custom_install_folder, make_schema, make_table, make_random, checks_location, quarantine + ) + return installation_ctx_custom_install_folder, run_config + + def delete(resource) -> None: + ctx, run_config = resource + checks_location = f"{ctx.installation.install_folder()}/{run_config.checks_location}" + ws.workspace.delete(checks_location) + + yield from factory("workflows", lambda **kw: create(spark, **kw), delete) + + def _setup_workflows_deps( ctx, make_schema, diff --git a/tests/integration/test_e2e_workflow.py b/tests/integration/test_e2e_workflow.py index 4bfb4f706..2d1517826 100644 --- a/tests/integration/test_e2e_workflow.py +++ b/tests/integration/test_e2e_workflow.py @@ -40,3 +40,26 @@ def test_e2e_workflow_serverless(ws, spark, setup_serverless_workflows, expected quarantine_df = spark.table(run_config.quarantine_config.location) assert quarantine_df.count() > 0, "Output table is empty" + + +def test_e2e_workflow_with_custom_install_folder( + ws, spark, setup_workflows_with_custom_install_folder, expected_quality_checking_output +): + installation_ctx, run_config = setup_workflows_with_custom_install_folder() + + installation_ctx.deployed_workflows.run_workflow("e2e", run_config.name) + + config = InstallationChecksStorageConfig( + run_config_name=run_config.name, + assume_user=True, + product_name=installation_ctx.installation.product(), + install_folder=installation_ctx.installation.install_folder(), + ) + checks = DQEngine(ws, spark).load_checks(config=config) + assert checks, "Checks were not loaded correctly" + + checked_df = spark.table(run_config.output_config.location) + input_df = spark.table(run_config.input_config.location) + + # this is sanity check only, we cannot predict the exact output as it depends on the generated rules + assert checked_df.count() == input_df.count(), "Output table is empty" diff --git a/tests/integration/test_profiler_workflow.py b/tests/integration/test_profiler_workflow.py index ffec805f3..932af401d 100644 --- a/tests/integration/test_profiler_workflow.py +++ b/tests/integration/test_profiler_workflow.py @@ -82,3 +82,24 @@ def test_profiler_workflow_serverless(ws, spark, setup_serverless_workflows): install_folder = installation_ctx.installation.install_folder() status = ws.workspace.get_status(f"{install_folder}/{run_config.profiler_config.summary_stats_file}") assert status, f"Profile summary stats file {run_config.profiler_config.summary_stats_file} does not exist." + + +def test_profiler_workflow_with_custom_install_folder(ws, spark, setup_workflows_with_custom_install_folder): + installation_ctx, run_config = setup_workflows_with_custom_install_folder() + + installation_ctx.deployed_workflows.run_workflow("profiler", run_config.name) + + config = InstallationChecksStorageConfig( + run_config_name=run_config.name, + assume_user=True, + product_name=installation_ctx.installation.product(), + install_folder=installation_ctx.installation.install_folder(), + ) + + dq_engine = DQEngine(ws, spark) + checks = dq_engine.load_checks(config=config) + assert checks, "Checks were not loaded correctly" + + install_folder = installation_ctx.installation.install_folder() + status = ws.workspace.get_status(f"{install_folder}/{run_config.profiler_config.summary_stats_file}") + assert status, f"Profile summary stats file {run_config.profiler_config.summary_stats_file} does not exist." diff --git a/tests/integration/test_quality_checker_workflow.py b/tests/integration/test_quality_checker_workflow.py index 494432036..65c74025f 100644 --- a/tests/integration/test_quality_checker_workflow.py +++ b/tests/integration/test_quality_checker_workflow.py @@ -29,6 +29,17 @@ def test_quality_checker_workflow_serverless(ws, spark, setup_serverless_workflo assert_df_equality(checked_df, expected_quality_checking_output, ignore_nullable=True) +def test_quality_checker_workflow_with_custom_install_folder( + ws, spark, setup_workflows_with_custom_install_folder, expected_quality_checking_output +): + installation_ctx, run_config = setup_workflows_with_custom_install_folder(checks=True) + + installation_ctx.deployed_workflows.run_workflow("quality-checker", run_config.name) + + checked_df = spark.table(run_config.output_config.location) + assert_df_equality(checked_df, expected_quality_checking_output, ignore_nullable=True) + + def test_quality_checker_workflow_streaming(ws, spark, setup_serverless_workflows, expected_quality_checking_output): installation_ctx, run_config = setup_serverless_workflows(checks=True, is_streaming=True) From 9bb2a7c509dc913d7c23644139d1f3395ee09a01 Mon Sep 17 00:00:00 2001 From: Marcin Wojtyczka Date: Fri, 19 Sep 2025 11:06:24 +0200 Subject: [PATCH 16/31] added integration tests --- .../dqx/docs/guide/quality_checks_storage.mdx | 21 ++++++--- docs/dqx/docs/installation.mdx | 23 +++++----- docs/dqx/docs/reference/engine.mdx | 44 +++++++++---------- src/databricks/labs/dqx/config.py | 7 ++- src/databricks/labs/dqx/config_loader.py | 10 ++--- src/databricks/labs/dqx/engine.py | 18 +++++--- tests/integration/conftest.py | 2 +- tests/integration/test_config.py | 17 ++++++- tests/integration/test_e2e_workflow.py | 4 +- tests/integration/test_profiler_workflow.py | 4 +- .../test_quality_checker_workflow.py | 4 +- .../integration/test_save_results_in_table.py | 37 ++++++++++++++++ 12 files changed, 132 insertions(+), 59 deletions(-) diff --git a/docs/dqx/docs/guide/quality_checks_storage.mdx b/docs/dqx/docs/guide/quality_checks_storage.mdx index a1f858f61..0bf7c7a8f 100644 --- a/docs/dqx/docs/guide/quality_checks_storage.mdx +++ b/docs/dqx/docs/guide/quality_checks_storage.mdx @@ -12,11 +12,22 @@ import TabItem from '@theme/TabItem'; DQX provides flexible methods to load and save quality checks (rules) defined as metadata (a list of dictionaries) from different storage backends, making it easier to manage, share, and reuse checks across workflows and environments. Saving and loading methods accept a storage backend configuration as input. The following backend configuration are currently supported: -- `FileChecksStorageConfig`: local files (JSON/YAML), or workspace files if invoked from Databricks notebook or job -- `WorkspaceFileChecksStorageConfig`: workspace files (JSON/YAML) using absolute paths -- `VolumeFileChecksStorageConfig`: Unity Catalog volumes (JSON/YAML file) -- `TableChecksStorageConfig`: Unity Catalog tables -- `InstallationChecksStorageConfig`: installation-managed location from the run config, ignores location and infers it from `checks_location` in the run config +* `FileChecksStorageConfig`: local files (JSON/YAML), or workspace files if invoked from Databricks notebook or job. Containing fields: + * `location`: absolute or relative file path in the local filesystem (JSON or YAML); also works with absolute or relative workspace file paths if invoked from Databricks notebook or job. +* `WorkspaceFileChecksStorageConfig`: workspace files (JSON/YAML) using absolute paths. Containing fields: + * `location`: absolute workspace file path (JSON or YAML). +* `TableChecksStorageConfig`: Unity Catalog tables. Containing fields: + * `location`: table fully qualified name. + * `run_config_name`: (optional) run configuration name to load (use "default" if not provided). + * `mode`: (optional) write mode for saving checks (`overwrite` or `append`, default is `overwrite`). The `overwrite` mode will only replace checks for the specific run config and not all checks in the table. +* `VolumeFileChecksStorageConfig`: Unity Catalog volumes (JSON/YAML file). Containing fields: + * `location`: Unity Catalog Volume file path (JSON or YAML). +* `InstallationChecksStorageConfig`: installation-managed location from the run config, ignores location and infers it from `checks_location` in the run config. Containing fields: + * `location` (optional): automatically set based on the `checks_location` field from the run configuration. + * `install_folder`: (optional) installation folder where DQX is installed, only required when custom installation folder is used. + * `run_config_name` (optional) - run configuration name to load (use "default" if not provided). + * `product_name`: (optional) name of the product (use "dqx" if not provided). + * `assume_user`: (optional) if True, assume user installation, otherwise global installation (skipped if `install_folder` is provided). You can find details on how to define checks [here](/docs/guide/quality_checks_definition). diff --git a/docs/dqx/docs/installation.mdx b/docs/dqx/docs/installation.mdx index 9d20dd345..0b1854f91 100644 --- a/docs/dqx/docs/installation.mdx +++ b/docs/dqx/docs/installation.mdx @@ -82,8 +82,7 @@ Install a specific version of DQX in your Databricks workspace via Databricks CL databricks labs install dqx@v0.8.0 ``` -You'll be prompted to select a [configuration profile](https://docs.databricks.com/en/dev-tools/auth.html#databricks-client-unified-authentication) created by `databricks auth login` command, -and other configuration options. +You'll be prompted to select a [configuration profile](https://docs.databricks.com/en/dev-tools/auth.html#databricks-client-unified-authentication) created by `databricks auth login` command, and other configuration options. The cli command will install the following components in the workspace installation folder: - A Python [wheel file](https://peps.python.org/pep-0427/) with the library packaged. @@ -100,19 +99,23 @@ Alternatively, you can override the cluster configuration for each workflow in t #### Installation Options -DQX offers flexible installation options. By default, DQX is installed in the user home directory (under `/Users//.dqx`). +DQX offers flexible installation options. By default, DQX is installed in the user home directory under `/Users//.dqx`. +You can also install DQX in a global folder or any custom workspace folder. + +**Environment Variable Override:** + +You can force global or user installation using the 'DQX_FORCE_INSTALL' environment variable: + * `DQX_FORCE_INSTALL=global databricks labs install dqx`: forces installation to `/Applications/dqx` + * `DQX_FORCE_INSTALL=user databricks labs install dqx`: forces installation to `/Users//.dqx` (default behaviour) **Custom Workspace Folder:** + If you provide a custom path during installation (e.g., `/Shared/dqx-team` or `/Users/shared-user/dqx-project`), DQX will be installed there. +You will be prompted to optionally enter a workspace path when installing DQX. The custom folder installation is required when using [Group Assigned cluster](https://docs.databricks.com/aws/en/compute/group-access), as the concept of a user home directory does not exist in this setup. -**Environment Variable Override:** -You can also force global or user installation using the 'DQX_FORCE_INSTALL' environment variable: - * `DQX_FORCE_INSTALL=global databricks labs install dqx`: forces installation to `/Applications/dqx` - * `DQX_FORCE_INSTALL=user databricks labs install dqx`: forces installation to `/Users//.dqx` (default behaviour) - If a custom folder is provided during installation, the installation folder will take precedence over any environment variables (e.g. `DQX_FORCE_INSTALL`). @@ -123,9 +126,9 @@ DQX configuration file can contain multiple run configurations for different pip The configuration file is created in the installation directory depending on installation options (see above): - User home (default): `/Users//.dqx/config.yml` - Global: `/Applications/dqx/config.yml` (when DQX_FORCE_INSTALL=global) -- Custom folder: `/config.yml` +- Custom folder: `/config.yml` (when provided during installation) -The "default" run configuration is created during the installation. When DQX is upgraded, the configuration is preserved. +A "default" run configuration is created during the installation. When DQX is upgraded, the configuration is preserved. The configuration can be updated / extended manually by the user after the installation. Each run config defines configuration for one specific input and output location. Open the configuration file: diff --git a/docs/dqx/docs/reference/engine.mdx b/docs/dqx/docs/reference/engine.mdx index 8327b7f03..8195a433b 100644 --- a/docs/dqx/docs/reference/engine.mdx +++ b/docs/dqx/docs/reference/engine.mdx @@ -49,20 +49,20 @@ The following table outlines the available methods of the `DQEngine` and their f
**Available DQX engine methods** -| Method | Description | Arguments | Supports local execution | -| ---------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------- | -| `apply_checks` | Applies quality checks to the DataFrame and returns a DataFrame with result columns. | `df`: DataFrame to check; `checks`: List of checks defined using DQX classes, each check is an instance of the DQRule class; `ref_dfs`: Reference dataframes to use in the checks, if applicable. | Yes | -| `apply_checks_and_split` | Applies quality checks to the DataFrame and returns valid and invalid (quarantine) DataFrames with result columns. | `df`: DataFrame to check; `checks`: List of checks defined using DQX classes, each check is an instance of the DQRule class; `ref_dfs`: Reference dataframes to use in the checks, if applicable. | Yes | -| `apply_checks_and_save_in_table` | Applies quality checks using DQRule objects and writes results to valid and invalid Delta table(s) with result columns. | `input_config`: `InputConfig` object with the table name and options for reading the input data; `checks`: List of checks defined using DQX classes, each check is an instance of the DQRule class; `output_config`: `OutputConfig` object with the table name, output mode, and options for the output data; `quarantine_config`: `OutputConfig` object with the table name, output mode, and options for the quarantine data - if provided, data will be split; `ref_dfs`: Reference dataframes to use in the checks, if applicable. | No | -| `apply_checks_by_metadata` | Applies quality checks defined as a dictionary to the DataFrame and returns a DataFrame with result columns. | `df`: DataFrame to check; `checks`: List of checks defined as dictionary; `custom_check_functions`: (optional) dictionary with custom check functions (e.g., globals() of the calling module); `ref_dfs`: Reference dataframes to use in the checks, if applicable. | Yes | -| `apply_checks_by_metadata_and_split` | Applies quality checks defined as a dictionary and returns valid and invalid (quarantine) DataFrames. | `df`: DataFrame to check; `checks`: List of checks defined as dictionary; `custom_check_functions`: (optional) dictionary with custom check functions (e.g., globals() of the calling module); `ref_dfs`: Reference dataframes to use in the checks, if applicable. | Yes | -| `apply_checks_by_metadata_and_save_in_table` | Applies quality checks defined as a dictionary and writes results to valid and invalid Delta table(s) with result columns. | `input_config`: `InputConfig` object with the table name and options for reading the input data; `checks`: List of checks defined as dictionary; `output_config`: `OutputConfig` object with the table name, output mode, and options for the output data; `quarantine_config`: `OutputConfig` object with the table name, output mode, and options for the quarantine data - if provided, data will be split; `custom_check_functions`: (optional) dictionary with custom check functions; `ref_dfs`: Reference dataframes to use in the checks, if applicable. | No | -| `validate_checks` | Validates the provided quality checks to ensure they conform to the expected structure and types. | `checks`: List of checks to validate; `custom_check_functions`: (optional) dictionary of custom check functions that can be used; `validate_custom_check_functions`: (optional) if set to True, validates custom check functions (defaults to True). | Yes | -| `get_invalid` | Retrieves records from the DataFrame that violate data quality checks (records with warnings and errors). | `df`: Input DataFrame. | Yes | -| `get_valid` | Retrieves records from the DataFrame that pass all data quality checks. | `df`: Input DataFrame. | Yes | -| `load_checks` | Loads quality rules (checks) from storage backend. Multiple storage backends are supported including tables, files or workspace files, installation-managed sources where the location is inferred automatically from run config. | `config`: Configuration for loading checks from a storage backend, i.e. `FileChecksStorageConfig`: file in a local filesystem (YAML or JSON), or workspace files if invoked from Databricks notebook or job; `WorkspaceFileChecksStorageConfig`: file in a workspace (YAML or JSON) using absolute paths; `VolumeFileChecksStorageConfig`: file in a Unity Catalog Volume (YAML or JSON); `TableChecksStorageConfig`: a table; `InstallationChecksStorageConfig`: installation-managed storage backend, using the `checks_location` field from the run configuration. See more details below. | Yes (only with `FileChecksStorageConfig`) | -| `save_checks` | Saves quality rules (checks) to storage backend. Multiple storage backends are supported including tables, files or workspace files, installation-managed targets where the location is inferred automatically from run config. | `checks`: List of checks defined as dictionary; `config`: Configuration for saving checks in a storage backend, i.e. `FileChecksStorageConfig`: file in a local filesystem (YAML or JSON), or workspace files if invoked from Databricks notebook or job; `WorkspaceFileChecksStorageConfig`: file in a workspace (YAML or JSON); `VolumeFileChecksStorageConfig`: file in a Unity Catalog Volume (YAML or JSON); `TableChecksStorageConfig`: a table; `InstallationChecksStorageConfig`: storage defined in the installation context, using the `checks_location` field from the run configuration. See more details below. | Yes (only with `FileChecksStorageConfig`) | -| `save_results_in_table` | Save quality checking results in delta table(s). | `output_df`: (optional) Dataframe containing the output data; `quarantine_df`: (optional) Dataframe containing the output data; `output_config`: `OutputConfig` object with the table name, output mode, and options for the output data; `quarantine_config`: `OutputConfig` object with the table name, output mode, and options for the quarantine data - if provided, data will be split; `run_config_name`: Name of the run config to use; `assume_user`: If True, assume user installation. | No | +| Method | Description | Arguments | Supports local execution | +| ---------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------- | +| `apply_checks` | Applies quality checks to the DataFrame and returns a DataFrame with result columns. | `df`: DataFrame to check; `checks`: List of checks defined using DQX classes, each check is an instance of the DQRule class; `ref_dfs`: Reference dataframes to use in the checks, if applicable. | Yes | +| `apply_checks_and_split` | Applies quality checks to the DataFrame and returns valid and invalid (quarantine) DataFrames with result columns. | `df`: DataFrame to check; `checks`: List of checks defined using DQX classes, each check is an instance of the DQRule class; `ref_dfs`: Reference dataframes to use in the checks, if applicable. | Yes | +| `apply_checks_and_save_in_table` | Applies quality checks using DQRule objects and writes results to valid and invalid Delta table(s) with result columns. | `input_config`: `InputConfig` object with the table name and options for reading the input data; `checks`: List of checks defined using DQX classes, each check is an instance of the DQRule class; `output_config`: `OutputConfig` object with the table name, output mode, and options for the output data; `quarantine_config`: `OutputConfig` object with the table name, output mode, and options for the quarantine data - if provided, data will be split; `ref_dfs`: Reference dataframes to use in the checks, if applicable. | No | +| `apply_checks_by_metadata` | Applies quality checks defined as a dictionary to the DataFrame and returns a DataFrame with result columns. | `df`: DataFrame to check; `checks`: List of checks defined as dictionary; `custom_check_functions`: (optional) dictionary with custom check functions (e.g., globals() of the calling module); `ref_dfs`: Reference dataframes to use in the checks, if applicable. | Yes | +| `apply_checks_by_metadata_and_split` | Applies quality checks defined as a dictionary and returns valid and invalid (quarantine) DataFrames. | `df`: DataFrame to check; `checks`: List of checks defined as dictionary; `custom_check_functions`: (optional) dictionary with custom check functions (e.g., globals() of the calling module); `ref_dfs`: Reference dataframes to use in the checks, if applicable. | Yes | +| `apply_checks_by_metadata_and_save_in_table` | Applies quality checks defined as a dictionary and writes results to valid and invalid Delta table(s) with result columns. | `input_config`: `InputConfig` object with the table name and options for reading the input data; `checks`: List of checks defined as dictionary; `output_config`: `OutputConfig` object with the table name, output mode, and options for the output data; `quarantine_config`: `OutputConfig` object with the table name, output mode, and options for the quarantine data - if provided, data will be split; `custom_check_functions`: (optional) dictionary with custom check functions; `ref_dfs`: Reference dataframes to use in the checks, if applicable. | No | +| `validate_checks` | Validates the provided quality checks to ensure they conform to the expected structure and types. | `checks`: List of checks to validate; `custom_check_functions`: (optional) dictionary of custom check functions that can be used; `validate_custom_check_functions`: (optional) if set to True, validates custom check functions (defaults to True). | Yes | +| `get_invalid` | Retrieves records from the DataFrame that violate data quality checks (records with warnings and errors). | `df`: Input DataFrame. | Yes | +| `get_valid` | Retrieves records from the DataFrame that pass all data quality checks. | `df`: Input DataFrame. | Yes | +| `load_checks` | Loads quality rules (checks) from storage backend. Multiple storage backends are supported including tables, files or workspace files, installation-managed sources where the location is inferred automatically from run config. | `config`: Configuration for loading checks from a storage backend, i.e. `FileChecksStorageConfig`: file in a local filesystem (YAML or JSON), or workspace files if invoked from Databricks notebook or job; `WorkspaceFileChecksStorageConfig`: file in a workspace (YAML or JSON) using absolute paths; `VolumeFileChecksStorageConfig`: file in a Unity Catalog Volume (YAML or JSON); `TableChecksStorageConfig`: a table; `InstallationChecksStorageConfig`: installation-managed storage backend, using the `checks_location` field from the run configuration. See more details below. | Yes (only with `FileChecksStorageConfig`) | +| `save_checks` | Saves quality rules (checks) to storage backend. Multiple storage backends are supported including tables, files or workspace files, installation-managed targets where the location is inferred automatically from run config. | `checks`: List of checks defined as dictionary; `config`: Configuration for saving checks in a storage backend, i.e. `FileChecksStorageConfig`: file in a local filesystem (YAML or JSON), or workspace files if invoked from Databricks notebook or job; `WorkspaceFileChecksStorageConfig`: file in a workspace (YAML or JSON); `VolumeFileChecksStorageConfig`: file in a Unity Catalog Volume (YAML or JSON); `TableChecksStorageConfig`: a table; `InstallationChecksStorageConfig`: storage defined in the installation context, using the `checks_location` field from the run configuration. See more details below. | Yes (only with `FileChecksStorageConfig`) | +| `save_results_in_table` | Save quality checking results in delta table(s). | `output_df`: (optional) Dataframe containing the output data; `quarantine_df`: (optional) Dataframe containing the output data; `output_config`: `OutputConfig` object with the table name, output mode, and options for the output data; `quarantine_config`: `OutputConfig` object with the table name, output mode, and options for the quarantine data - if provided, data will be split; `run_config_name`: Name of the run config to use; `install_folder`: (optional) installation folder where DQX is installed, only required when custom installation folder is used; `assume_user`: (optional) If True, assume user installation, otherwise global installation (skipped if `install_folder` is provided). | No | The 'Supports local execution' in the above table indicates which methods can be used for local testing without a Databricks workspace (see the usage in [local testing section](/docs/reference/testing/#local-execution-and-testing-with-dqengine)). @@ -82,21 +82,21 @@ The 'Supports local execution' in the above table indicates which methods can be Supported storage backend configurations (implementations of `BaseChecksStorageConfig`) for `load_checks` and `save_checks` methods: * `FileChecksStorageConfig` can be used to save or load checks from a local filesystem, or workspace file if invoked from Databricks notebook or job, with fields: - * `location`: absolute or relative file path in the local filesystem (JSON or YAML); also works with absolute or relative workspace file paths if invoked from Databricks notebook or job + * `location`: absolute or relative file path in the local filesystem (JSON or YAML); also works with absolute or relative workspace file paths if invoked from Databricks notebook or job. * `WorkspaceFileChecksStorageConfig` can be used to save or load checks from a workspace file, with fields: - * `location`: absolute workspace file path (JSON or YAML) + * `location`: absolute workspace file path (JSON or YAML). * `TableChecksStorageConfig` can be used to save or load checks from a table, with fields: - * `location`: table fully qualified name - * `run_config_name`: (optional) run configuration name to load (use "default" if not provided) + * `location`: table fully qualified name. + * `run_config_name`: (optional) run configuration name to load (use "default" if not provided). * `mode`: (optional) write mode for saving checks (`overwrite` or `append`, default is `overwrite`). The `overwrite` mode will only replace checks for the specific run config and not all checks in the table. * `VolumeFileChecksStorageConfig` can be used to save or load checks from a Unity Catalog Volume file, with fields: - * `location`: Unity Catalog Volume file path (JSON or YAML) + * `location`: Unity Catalog Volume file path (JSON or YAML). * `InstallationChecksStorageConfig` can be used to save or load checks from workspace installation, with fields: * `location` (optional): automatically set based on the `checks_location` field from the run configuration. - * `install_folder`: (optional) installation folder where DQX is installed, only required when custom installation folder is used + * `install_folder`: (optional) installation folder where DQX is installed, only required when custom installation folder is used. * `run_config_name` (optional) - run configuration name to load (use "default" if not provided). - * `product_name`: (optional) name of the product/installation directory - * `assume_user`: (optional) if True, assume user installation + * `product_name`: (optional) name of the product (use "dqx" if not provided). + * `assume_user`: (optional) if True, assume user installation, otherwise global installation (skipped if `install_folder` is provided). For details on how to prepare reference DataFrames (`ref_dfs`) and custom check function mapping (`custom_check_functions`) refer to [Quality Checks Reference](/docs/reference/quality_checks).
diff --git a/src/databricks/labs/dqx/config.py b/src/databricks/labs/dqx/config.py index fd0be4f8d..d2df13989 100644 --- a/src/databricks/labs/dqx/config.py +++ b/src/databricks/labs/dqx/config.py @@ -220,14 +220,13 @@ class InstallationChecksStorageConfig( product_name: The product name for retrieving checks from the installation (default is 'dqx'). assume_user: Whether to assume the user is the owner of the checks (default is True). install_folder: The installation folder where DQX is installed. - Default to /Users//.dqx if not provided. + DQX will be installed in a default directory if no custom folder is provided: + * User's home directory: "/Users//.dqx" + * Global directory if `DQX_FORCE_INSTALL=global`: "/Applications/dqx" """ location: str = "installation" # retrieved from the installation config run_config_name: str = "default" # to retrieve run config product_name: str = "dqx" assume_user: bool = True - # DQX will be installed in a default directory if no custom folder is provided: - # * Default: "/Users//.dqx" - # * if `DQX_FORCE_INSTALL=global`: "/Applications/dqx" install_folder: str | None = None diff --git a/src/databricks/labs/dqx/config_loader.py b/src/databricks/labs/dqx/config_loader.py index dc8e446f0..6bad58771 100644 --- a/src/databricks/labs/dqx/config_loader.py +++ b/src/databricks/labs/dqx/config_loader.py @@ -24,10 +24,10 @@ def load_run_config( Load run configuration from the installation. Args: - run_config_name: name of the run configuration to use - install_folder: optional installation folder - assume_user: if True, assume user installation - product_name: name of the product + run_config_name: Name of the run configuration to use. + install_folder: Custom workspace installation folder. Required if DQX is installed in a custom folder. + assume_user: Whether to assume a per-user installation when loading the run configuration (True as default, skipped if install_folder is provided). + product_name: Product/installation identifier used to resolve installation paths for config loading in install_folder is not provided ("dqx" as default). """ installation = self.get_installation(assume_user, product_name, install_folder) return self._load_run_config(installation, run_config_name) @@ -57,7 +57,7 @@ def get_installation(self, assume_user: bool, product_name: str, install_folder: @staticmethod def get_custom_installation(ws: WorkspaceClient, product_name: str, install_folder: str) -> Installation: """ - Creates an Installation instance for a custom folder, similar to assume_user_home and assume_global. + Creates an Installation instance for a custom installation folder, similar to assume_user_home and assume_global. This ensures the custom folder is created in the workspace when the installation is accessed. Args: diff --git a/src/databricks/labs/dqx/engine.py b/src/databricks/labs/dqx/engine.py index 34024caba..9f1525a05 100644 --- a/src/databricks/labs/dqx/engine.py +++ b/src/databricks/labs/dqx/engine.py @@ -608,6 +608,7 @@ def save_results_in_table( run_config_name: str | None = "default", product_name: str = "dqx", assume_user: bool = True, + install_folder: str | None = None, ): """Persist result DataFrames using explicit configs or the named run configuration. @@ -620,23 +621,30 @@ def save_results_in_table( output_df: DataFrame with valid rows to be saved (optional). quarantine_df: DataFrame with invalid rows to be saved (optional). output_config: Configuration describing where/how to write the valid rows. If omitted, falls back to the run config. - quarantine_config: Configuration describing where/how to write the invalid rows. If omitted, falls back to the run config. + quarantine_config: Configuration describing where/how to write the invalid rows (optional). If omitted, falls back to the run config. run_config_name: Name of the run configuration to load when a config parameter is omitted. - product_name: Product/installation identifier used to resolve installation paths for config loading. - assume_user: Whether to assume a per-user installation when loading the run configuration. + product_name: Product/installation identifier used to resolve installation paths for config loading in install_folder is not provided ("dqx" as default). + assume_user: Whether to assume a per-user installation when loading the run configuration (True as default, skipped if install_folder is provided). + install_folder: Custom workspace installation folder. Required if DQX is installed in a custom folder. Returns: None """ if output_df is not None and output_config is None: run_config = self._run_config_loader.load_run_config( - run_config_name=run_config_name, assume_user=assume_user, product_name=product_name + run_config_name=run_config_name, + assume_user=assume_user, + product_name=product_name, + install_folder=install_folder, ) output_config = run_config.output_config if quarantine_df is not None and quarantine_config is None: run_config = self._run_config_loader.load_run_config( - run_config_name=run_config_name, assume_user=assume_user, product_name=product_name + run_config_name=run_config_name, + assume_user=assume_user, + product_name=product_name, + install_folder=install_folder, ) quarantine_config = run_config.quarantine_config diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 33742c213..2bcca8818 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -92,7 +92,7 @@ def delete(resource) -> None: @pytest.fixture -def setup_workflows_with_custom_install_folder( +def setup_workflows_with_custom_folder( ws, spark, installation_ctx_custom_install_folder, make_schema, make_table, make_random ): """ diff --git a/tests/integration/test_config.py b/tests/integration/test_config.py index 54558d6a1..b336e0ab0 100644 --- a/tests/integration/test_config.py +++ b/tests/integration/test_config.py @@ -7,7 +7,7 @@ from databricks.labs.dqx.config_loader import RunConfigLoader -def test_load_run_config_from_user_installation(ws, installation_ctx, spark): +def test_load_run_config_from_user_installation(ws, installation_ctx): installation_ctx.installation.save(installation_ctx.config) product_name = installation_ctx.product_info.product_name() @@ -34,6 +34,21 @@ def test_load_run_config_from_global_installation(ws, installation_ctx): assert run_config == expected_run_config +def test_load_run_config_from_custom_folder_installation(ws, installation_ctx_custom_install_folder): + installation_ctx_custom_install_folder.installation.save(installation_ctx_custom_install_folder.config) + product_name = installation_ctx_custom_install_folder.product_info.product_name() + + run_config = RunConfigLoader(ws).load_run_config( + run_config_name="default", + assume_user=True, + product_name=product_name, + install_folder=installation_ctx_custom_install_folder.install_folder, + ) + expected_run_config = installation_ctx_custom_install_folder.config.get_run_config("default") + + assert run_config == expected_run_config + + def test_get_custom_installation(ws, make_directory): product_info = ProductInfo.for_testing(WorkspaceConfig) custom_folder = str(make_directory().absolute()) diff --git a/tests/integration/test_e2e_workflow.py b/tests/integration/test_e2e_workflow.py index 2d1517826..29dd4498d 100644 --- a/tests/integration/test_e2e_workflow.py +++ b/tests/integration/test_e2e_workflow.py @@ -43,9 +43,9 @@ def test_e2e_workflow_serverless(ws, spark, setup_serverless_workflows, expected def test_e2e_workflow_with_custom_install_folder( - ws, spark, setup_workflows_with_custom_install_folder, expected_quality_checking_output + ws, spark, setup_workflows_with_custom_folder, expected_quality_checking_output ): - installation_ctx, run_config = setup_workflows_with_custom_install_folder() + installation_ctx, run_config = setup_workflows_with_custom_folder() installation_ctx.deployed_workflows.run_workflow("e2e", run_config.name) diff --git a/tests/integration/test_profiler_workflow.py b/tests/integration/test_profiler_workflow.py index 932af401d..cb426bce3 100644 --- a/tests/integration/test_profiler_workflow.py +++ b/tests/integration/test_profiler_workflow.py @@ -84,8 +84,8 @@ def test_profiler_workflow_serverless(ws, spark, setup_serverless_workflows): assert status, f"Profile summary stats file {run_config.profiler_config.summary_stats_file} does not exist." -def test_profiler_workflow_with_custom_install_folder(ws, spark, setup_workflows_with_custom_install_folder): - installation_ctx, run_config = setup_workflows_with_custom_install_folder() +def test_profiler_workflow_with_custom_install_folder(ws, spark, setup_workflows_with_custom_folder): + installation_ctx, run_config = setup_workflows_with_custom_folder() installation_ctx.deployed_workflows.run_workflow("profiler", run_config.name) diff --git a/tests/integration/test_quality_checker_workflow.py b/tests/integration/test_quality_checker_workflow.py index 65c74025f..d6bf31827 100644 --- a/tests/integration/test_quality_checker_workflow.py +++ b/tests/integration/test_quality_checker_workflow.py @@ -30,9 +30,9 @@ def test_quality_checker_workflow_serverless(ws, spark, setup_serverless_workflo def test_quality_checker_workflow_with_custom_install_folder( - ws, spark, setup_workflows_with_custom_install_folder, expected_quality_checking_output + ws, spark, setup_workflows_with_custom_folder, expected_quality_checking_output ): - installation_ctx, run_config = setup_workflows_with_custom_install_folder(checks=True) + installation_ctx, run_config = setup_workflows_with_custom_folder(checks=True) installation_ctx.deployed_workflows.run_workflow("quality-checker", run_config.name) diff --git a/tests/integration/test_save_results_in_table.py b/tests/integration/test_save_results_in_table.py index f8d9c6602..a0454ad1c 100644 --- a/tests/integration/test_save_results_in_table.py +++ b/tests/integration/test_save_results_in_table.py @@ -287,6 +287,43 @@ def test_save_results_in_table_in_user_installation_missing_output_and_quarantin ), "Quarantine table should not have been saved" +def test_save_results_in_table_in_custom_folder_installation( + ws, spark, installation_ctx_custom_install_folder, make_schema, make_random +): + catalog_name = "main" + schema = make_schema(catalog_name=catalog_name) + output_table = f"{catalog_name}.{schema.name}.{make_random(6).lower()}" + quarantine_table = f"{catalog_name}.{schema.name}.{make_random(6).lower()}" + + config = installation_ctx_custom_install_folder.config + run_config = config.get_run_config() + run_config.output_config = OutputConfig(location=output_table) + run_config.quarantine_config = OutputConfig(location=quarantine_table) + installation_ctx_custom_install_folder.installation.save(installation_ctx_custom_install_folder.config) + product_name = installation_ctx_custom_install_folder.product_info.product_name() + install_folder = installation_ctx_custom_install_folder.install_folder + + schema = "a: int, b: int" + output_df = spark.createDataFrame([[1, 2]], schema) + quarantine_df = spark.createDataFrame([[3, 4]], schema) + + engine = DQEngine(ws, spark) + engine.save_results_in_table( + output_df=output_df, + quarantine_df=quarantine_df, + run_config_name=run_config.name, + product_name=product_name, + assume_user=True, + install_folder=install_folder, + ) + + output_df_loaded = spark.table(output_table) + quarantine_df_loaded = spark.table(quarantine_table) + + assert_df_equality(output_df, output_df_loaded) + assert_df_equality(quarantine_df, quarantine_df_loaded) + + def test_save_streaming_results_in_table(ws, spark, make_schema, make_random, make_volume): catalog_name = "main" schema = make_schema(catalog_name=catalog_name) From 226714b6b8479e716935ff88342908c6898c2043 Mon Sep 17 00:00:00 2001 From: Marcin Wojtyczka Date: Fri, 19 Sep 2025 11:17:27 +0200 Subject: [PATCH 17/31] added integration tests --- tests/integration/conftest.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 2bcca8818..4e36ec60b 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -260,6 +260,7 @@ def _setup_quality_checks(ctx, spark, ws): config = InstallationChecksStorageConfig( location=checks_location, product_name=ctx.installation.product(), + install_folder=ctx.installation.install_folder(), ) InstallationChecksStorageHandler(ws, spark).save(checks=checks, config=config) From fa4712ea58f35ae3efe253e6a4b32f0441c89931 Mon Sep 17 00:00:00 2001 From: Marcin Wojtyczka Date: Fri, 19 Sep 2025 11:51:45 +0200 Subject: [PATCH 18/31] upgraded hatch version --- .github/workflows/acceptance.yml | 8 ++++---- .github/workflows/docs-release.yml | 2 +- .github/workflows/downstreams.yml | 2 +- .github/workflows/nightly.yml | 10 +++++----- .github/workflows/performance.yml | 2 +- .github/workflows/push.yml | 2 +- .github/workflows/release.yml | 2 +- tests/conftest.py | 2 +- 8 files changed, 15 insertions(+), 15 deletions(-) diff --git a/.github/workflows/acceptance.yml b/.github/workflows/acceptance.yml index 1409b1d88..206a44d4d 100644 --- a/.github/workflows/acceptance.yml +++ b/.github/workflows/acceptance.yml @@ -41,7 +41,7 @@ jobs: python-version: '3.12' - name: Install hatch - run: pip install hatch==1.9.4 + run: pip install hatch==1.13.0 - name: Run unit tests and generate test coverage report run: make test @@ -93,7 +93,7 @@ jobs: python-version: '3.12' - name: Install hatch - run: pip install hatch==1.9.4 + run: pip install hatch==1.13.0 - name: Run integration tests on serverless cluster uses: databrickslabs/sandbox/acceptance@acceptance/v0.4.4 @@ -125,7 +125,7 @@ jobs: python-version: '3.12' - name: Install hatch - run: pip install hatch==1.9.4 + run: pip install hatch==1.13.0 - name: Install Databricks CLI run: | @@ -177,7 +177,7 @@ jobs: python-version: '3.12' - name: Install hatch - run: pip install hatch==1.9.4 + run: pip install hatch==1.13.0 - name: Install Databricks CLI run: | diff --git a/.github/workflows/docs-release.yml b/.github/workflows/docs-release.yml index 052a9e591..01aef396a 100644 --- a/.github/workflows/docs-release.yml +++ b/.github/workflows/docs-release.yml @@ -28,7 +28,7 @@ jobs: - name: Install Hatch run: | - pip install hatch==1.9.4 + pip install hatch==1.13.0 - uses: actions/setup-node@v4 with: diff --git a/.github/workflows/downstreams.yml b/.github/workflows/downstreams.yml index 982f5019f..1e5f58b73 100644 --- a/.github/workflows/downstreams.yml +++ b/.github/workflows/downstreams.yml @@ -43,7 +43,7 @@ jobs: - name: Install toolchain run: | - pip install hatch==1.9.4 + pip install hatch==1.13.0 - name: Check downstream compatibility uses: databrickslabs/sandbox/downstreams@downstreams/v0.0.1 diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 23b06c7e0..28c503648 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -32,7 +32,7 @@ jobs: python-version: '3.12' - name: Install hatch - run: pip install hatch==1.9.4 + run: pip install hatch==1.13.0 - name: Run unit tests and generate test coverage report run: make test @@ -81,7 +81,7 @@ jobs: python-version: '3.12' - name: Install hatch - run: pip install hatch==1.9.4 + run: pip install hatch==1.13.0 - name: Run integration tests on serverless cluster uses: databrickslabs/sandbox/acceptance@acceptance/v0.4.4 @@ -113,7 +113,7 @@ jobs: python-version: '3.12' - name: Install hatch - run: pip install hatch==1.9.4 + run: pip install hatch==1.13.0 - name: Install Databricks CLI run: | @@ -164,7 +164,7 @@ jobs: python-version: '3.12' - name: Install hatch - run: pip install hatch==1.9.4 + run: pip install hatch==1.13.0 - name: Install Databricks CLI run: | @@ -217,7 +217,7 @@ jobs: python-version: '3.12' - name: Install hatch - run: pip install hatch==1.9.4 + run: pip install hatch==1.13.0 - name: Login to Azure for azure-cli authentication uses: azure/login@v2 diff --git a/.github/workflows/performance.yml b/.github/workflows/performance.yml index cda38de99..67cb5ef8d 100644 --- a/.github/workflows/performance.yml +++ b/.github/workflows/performance.yml @@ -43,7 +43,7 @@ jobs: cache-dependency-path: '**/pyproject.toml' - name: Install hatch - run: pip install hatch==1.9.4 + run: pip install hatch==1.13.0 - name: Login to Azure for azure-cli authentication uses: azure/login@v2 diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index 6f14655ef..e76a154bd 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -37,7 +37,7 @@ jobs: - name: Run unit tests run: | - pip install hatch==1.9.4 + pip install hatch==1.13.0 make test fmt: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 025718988..eef18d287 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -27,7 +27,7 @@ jobs: - name: Build wheels run: | - pip install hatch==1.9.4 + pip install hatch==1.13.0 hatch build - name: Github release diff --git a/tests/conftest.py b/tests/conftest.py index 5766c0846..37ca657b2 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -22,7 +22,7 @@ @pytest.fixture def debug_env_name(): - return "ws" # Specify the name of the debug environment from ~/.databricks/debug-env.json + return "ws2" # Specify the name of the debug environment from ~/.databricks/debug-env.json @pytest.fixture From 9ef88d40c6f14637a22b97af57afc9724b2dfa16 Mon Sep 17 00:00:00 2001 From: Marcin Wojtyczka Date: Fri, 19 Sep 2025 11:56:18 +0200 Subject: [PATCH 19/31] disable hatch filters for tests --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 74cf020e2..26615c5fb 100644 --- a/Makefile +++ b/Makefile @@ -20,7 +20,7 @@ fmt: hatch run extract_checks_examples test: - hatch run test + hatch run test --filter '{}' integration: hatch run integration From b19da0b738a9d1f38d61b698ecca80d1328bee59 Mon Sep 17 00:00:00 2001 From: Marcin Wojtyczka Date: Fri, 19 Sep 2025 11:57:24 +0200 Subject: [PATCH 20/31] disable hatch filters for tests --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 26615c5fb..dd686c8c0 100644 --- a/Makefile +++ b/Makefile @@ -20,7 +20,7 @@ fmt: hatch run extract_checks_examples test: - hatch run test --filter '{}' + hatch run test --filter "" integration: hatch run integration From 85ce3a7703ec7ef2ecf572394f892cb620c5e373 Mon Sep 17 00:00:00 2001 From: Marcin Wojtyczka Date: Fri, 19 Sep 2025 11:58:59 +0200 Subject: [PATCH 21/31] disable hatch filters for tests --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index dd686c8c0..811ba3021 100644 --- a/Makefile +++ b/Makefile @@ -20,7 +20,7 @@ fmt: hatch run extract_checks_examples test: - hatch run test --filter "" + hatch run test --filter='{}' integration: hatch run integration From a9c67d5b632b638166598f272fa43ed8fd0e0cc0 Mon Sep 17 00:00:00 2001 From: Marcin Wojtyczka Date: Fri, 19 Sep 2025 12:03:32 +0200 Subject: [PATCH 22/31] test --- .github/workflows/push.yml | 2 +- Makefile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index e76a154bd..2aa2cc3e8 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -37,7 +37,7 @@ jobs: - name: Run unit tests run: | - pip install hatch==1.13.0 + pip install hatch==1.7.0 make test fmt: diff --git a/Makefile b/Makefile index 811ba3021..74cf020e2 100644 --- a/Makefile +++ b/Makefile @@ -20,7 +20,7 @@ fmt: hatch run extract_checks_examples test: - hatch run test --filter='{}' + hatch run test integration: hatch run integration From 6bdeecaedc4d1196e87d775d3bad6757e9392b29 Mon Sep 17 00:00:00 2001 From: Marcin Wojtyczka Date: Fri, 19 Sep 2025 12:05:47 +0200 Subject: [PATCH 23/31] test --- .github/workflows/push.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index 2aa2cc3e8..160c09d55 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -35,9 +35,12 @@ jobs: cache-dependency-path: '**/pyproject.toml' python-version: ${{ matrix.pyVersion }} + - name: Debug env + run: env | grep HATCH || true + - name: Run unit tests run: | - pip install hatch==1.7.0 + pip install hatch==1.13.0 make test fmt: From d4908f1e99e418cdeec545159dc7b79308330334 Mon Sep 17 00:00:00 2001 From: Marcin Wojtyczka Date: Fri, 19 Sep 2025 12:07:39 +0200 Subject: [PATCH 24/31] test --- .github/workflows/push.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index 160c09d55..31cf43ab8 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -35,12 +35,11 @@ jobs: cache-dependency-path: '**/pyproject.toml' python-version: ${{ matrix.pyVersion }} - - name: Debug env - run: env | grep HATCH || true - - name: Run unit tests run: | - pip install hatch==1.13.0 + pip install --upgrade hatch==1.13.0 + which hatch + hatch --version make test fmt: From 720c6c3f66b3d39573caceac6cdf6e70fae0bbf7 Mon Sep 17 00:00:00 2001 From: Marcin Wojtyczka Date: Fri, 19 Sep 2025 12:09:55 +0200 Subject: [PATCH 25/31] test --- .github/workflows/push.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index 31cf43ab8..ef62363f3 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -39,7 +39,8 @@ jobs: run: | pip install --upgrade hatch==1.13.0 which hatch - hatch --version + hatch --version --verbose + pip freeze | grep click make test fmt: From 072420cfadccc93dfe15899bbf07a7f3ea9ff913 Mon Sep 17 00:00:00 2001 From: Marcin Wojtyczka Date: Fri, 19 Sep 2025 12:16:27 +0200 Subject: [PATCH 26/31] install specific click version --- .github/workflows/push.yml | 5 +---- pyproject.toml | 3 ++- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index ef62363f3..e76a154bd 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -37,10 +37,7 @@ jobs: - name: Run unit tests run: | - pip install --upgrade hatch==1.13.0 - which hatch - hatch --version --verbose - pip freeze | grep click + pip install hatch==1.13.0 make test fmt: diff --git a/pyproject.toml b/pyproject.toml index 22651cb63..39857c062 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -95,7 +95,8 @@ dependencies = [ "pyspark~=3.5.0", "dbldatagen~=0.4.0", "pyparsing~=3.2.3", - "jmespath~=1.0.1" + "jmespath~=1.0.1", + "click==8.2.1", # fix issue with hatch and click 8.3 ] python="3.12" # must match the version required by databricks-connect and python version on the test clusters From 08a1d337608dc0ad659d9ef90b0899e706ba6be8 Mon Sep 17 00:00:00 2001 From: Marcin Wojtyczka Date: Fri, 19 Sep 2025 12:19:46 +0200 Subject: [PATCH 27/31] install specific click version --- .github/workflows/acceptance.yml | 8 ++++---- .github/workflows/docs-release.yml | 2 +- .github/workflows/downstreams.yml | 2 +- .github/workflows/nightly.yml | 10 +++++----- .github/workflows/performance.yml | 2 +- .github/workflows/push.yml | 2 +- .github/workflows/release.yml | 2 +- pyproject.toml | 1 - 8 files changed, 14 insertions(+), 15 deletions(-) diff --git a/.github/workflows/acceptance.yml b/.github/workflows/acceptance.yml index 206a44d4d..99f882644 100644 --- a/.github/workflows/acceptance.yml +++ b/.github/workflows/acceptance.yml @@ -41,7 +41,7 @@ jobs: python-version: '3.12' - name: Install hatch - run: pip install hatch==1.13.0 + run: pip install hatch==1.13.0 click<8.3 - name: Run unit tests and generate test coverage report run: make test @@ -93,7 +93,7 @@ jobs: python-version: '3.12' - name: Install hatch - run: pip install hatch==1.13.0 + run: pip install hatch==1.13.0 click<8.3 - name: Run integration tests on serverless cluster uses: databrickslabs/sandbox/acceptance@acceptance/v0.4.4 @@ -125,7 +125,7 @@ jobs: python-version: '3.12' - name: Install hatch - run: pip install hatch==1.13.0 + run: pip install hatch==1.13.0 click<8.3 - name: Install Databricks CLI run: | @@ -177,7 +177,7 @@ jobs: python-version: '3.12' - name: Install hatch - run: pip install hatch==1.13.0 + run: pip install hatch==1.13.0 click<8.3 - name: Install Databricks CLI run: | diff --git a/.github/workflows/docs-release.yml b/.github/workflows/docs-release.yml index 01aef396a..0033d0384 100644 --- a/.github/workflows/docs-release.yml +++ b/.github/workflows/docs-release.yml @@ -28,7 +28,7 @@ jobs: - name: Install Hatch run: | - pip install hatch==1.13.0 + pip install hatch==1.13.0 click<8.3 - uses: actions/setup-node@v4 with: diff --git a/.github/workflows/downstreams.yml b/.github/workflows/downstreams.yml index 1e5f58b73..ab66963e8 100644 --- a/.github/workflows/downstreams.yml +++ b/.github/workflows/downstreams.yml @@ -43,7 +43,7 @@ jobs: - name: Install toolchain run: | - pip install hatch==1.13.0 + pip install hatch==1.13.0 click<8.3 - name: Check downstream compatibility uses: databrickslabs/sandbox/downstreams@downstreams/v0.0.1 diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 28c503648..1eaa28018 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -32,7 +32,7 @@ jobs: python-version: '3.12' - name: Install hatch - run: pip install hatch==1.13.0 + run: pip install hatch==1.13.0 click<8.3 - name: Run unit tests and generate test coverage report run: make test @@ -81,7 +81,7 @@ jobs: python-version: '3.12' - name: Install hatch - run: pip install hatch==1.13.0 + run: pip install hatch==1.13.0 click<8.3 - name: Run integration tests on serverless cluster uses: databrickslabs/sandbox/acceptance@acceptance/v0.4.4 @@ -113,7 +113,7 @@ jobs: python-version: '3.12' - name: Install hatch - run: pip install hatch==1.13.0 + run: pip install hatch==1.13.0 click<8.3 - name: Install Databricks CLI run: | @@ -164,7 +164,7 @@ jobs: python-version: '3.12' - name: Install hatch - run: pip install hatch==1.13.0 + run: pip install hatch==1.13.0 click<8.3 - name: Install Databricks CLI run: | @@ -217,7 +217,7 @@ jobs: python-version: '3.12' - name: Install hatch - run: pip install hatch==1.13.0 + run: pip install hatch==1.13.0 click<8.3 - name: Login to Azure for azure-cli authentication uses: azure/login@v2 diff --git a/.github/workflows/performance.yml b/.github/workflows/performance.yml index 67cb5ef8d..1e04ef0e7 100644 --- a/.github/workflows/performance.yml +++ b/.github/workflows/performance.yml @@ -43,7 +43,7 @@ jobs: cache-dependency-path: '**/pyproject.toml' - name: Install hatch - run: pip install hatch==1.13.0 + run: pip install hatch==1.13.0 click<8.3 - name: Login to Azure for azure-cli authentication uses: azure/login@v2 diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index e76a154bd..14d7e8f61 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -37,7 +37,7 @@ jobs: - name: Run unit tests run: | - pip install hatch==1.13.0 + pip install hatch==1.13.0 click<8.3 make test fmt: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index eef18d287..b3b690cf8 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -27,7 +27,7 @@ jobs: - name: Build wheels run: | - pip install hatch==1.13.0 + pip install hatch==1.13.0 click<8.3 hatch build - name: Github release diff --git a/pyproject.toml b/pyproject.toml index 39857c062..2d44164e0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -96,7 +96,6 @@ dependencies = [ "dbldatagen~=0.4.0", "pyparsing~=3.2.3", "jmespath~=1.0.1", - "click==8.2.1", # fix issue with hatch and click 8.3 ] python="3.12" # must match the version required by databricks-connect and python version on the test clusters From 9cc2d569b37df4591dfbb86b0d45929a98a8a8e9 Mon Sep 17 00:00:00 2001 From: Marcin Wojtyczka Date: Fri, 19 Sep 2025 12:21:16 +0200 Subject: [PATCH 28/31] install specific click version --- .github/workflows/acceptance.yml | 8 ++++---- .github/workflows/docs-release.yml | 2 +- .github/workflows/downstreams.yml | 2 +- .github/workflows/nightly.yml | 10 +++++----- .github/workflows/performance.yml | 2 +- .github/workflows/push.yml | 2 +- .github/workflows/release.yml | 2 +- 7 files changed, 14 insertions(+), 14 deletions(-) diff --git a/.github/workflows/acceptance.yml b/.github/workflows/acceptance.yml index 99f882644..40233904f 100644 --- a/.github/workflows/acceptance.yml +++ b/.github/workflows/acceptance.yml @@ -41,7 +41,7 @@ jobs: python-version: '3.12' - name: Install hatch - run: pip install hatch==1.13.0 click<8.3 + run: pip install "hatch==1.13.0" "click<8.3" - name: Run unit tests and generate test coverage report run: make test @@ -93,7 +93,7 @@ jobs: python-version: '3.12' - name: Install hatch - run: pip install hatch==1.13.0 click<8.3 + run: pip install "hatch==1.13.0" "click<8.3" - name: Run integration tests on serverless cluster uses: databrickslabs/sandbox/acceptance@acceptance/v0.4.4 @@ -125,7 +125,7 @@ jobs: python-version: '3.12' - name: Install hatch - run: pip install hatch==1.13.0 click<8.3 + run: pip install "hatch==1.13.0" "click<8.3" - name: Install Databricks CLI run: | @@ -177,7 +177,7 @@ jobs: python-version: '3.12' - name: Install hatch - run: pip install hatch==1.13.0 click<8.3 + run: pip install "hatch==1.13.0" "click<8.3" - name: Install Databricks CLI run: | diff --git a/.github/workflows/docs-release.yml b/.github/workflows/docs-release.yml index 0033d0384..07a6ce1fa 100644 --- a/.github/workflows/docs-release.yml +++ b/.github/workflows/docs-release.yml @@ -28,7 +28,7 @@ jobs: - name: Install Hatch run: | - pip install hatch==1.13.0 click<8.3 + pip install "hatch==1.13.0" "click<8.3" - uses: actions/setup-node@v4 with: diff --git a/.github/workflows/downstreams.yml b/.github/workflows/downstreams.yml index ab66963e8..c2551f6c2 100644 --- a/.github/workflows/downstreams.yml +++ b/.github/workflows/downstreams.yml @@ -43,7 +43,7 @@ jobs: - name: Install toolchain run: | - pip install hatch==1.13.0 click<8.3 + pip install "hatch==1.13.0" "click<8.3" - name: Check downstream compatibility uses: databrickslabs/sandbox/downstreams@downstreams/v0.0.1 diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 1eaa28018..2e9ea560a 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -32,7 +32,7 @@ jobs: python-version: '3.12' - name: Install hatch - run: pip install hatch==1.13.0 click<8.3 + run: pip install "hatch==1.13.0" "click<8.3" - name: Run unit tests and generate test coverage report run: make test @@ -81,7 +81,7 @@ jobs: python-version: '3.12' - name: Install hatch - run: pip install hatch==1.13.0 click<8.3 + run: pip install "hatch==1.13.0" "click<8.3" - name: Run integration tests on serverless cluster uses: databrickslabs/sandbox/acceptance@acceptance/v0.4.4 @@ -113,7 +113,7 @@ jobs: python-version: '3.12' - name: Install hatch - run: pip install hatch==1.13.0 click<8.3 + run: pip install "hatch==1.13.0" "click<8.3" - name: Install Databricks CLI run: | @@ -164,7 +164,7 @@ jobs: python-version: '3.12' - name: Install hatch - run: pip install hatch==1.13.0 click<8.3 + run: pip install "hatch==1.13.0" "click<8.3" - name: Install Databricks CLI run: | @@ -217,7 +217,7 @@ jobs: python-version: '3.12' - name: Install hatch - run: pip install hatch==1.13.0 click<8.3 + run: pip install "hatch==1.13.0" "click<8.3" - name: Login to Azure for azure-cli authentication uses: azure/login@v2 diff --git a/.github/workflows/performance.yml b/.github/workflows/performance.yml index 1e04ef0e7..7c1c9d650 100644 --- a/.github/workflows/performance.yml +++ b/.github/workflows/performance.yml @@ -43,7 +43,7 @@ jobs: cache-dependency-path: '**/pyproject.toml' - name: Install hatch - run: pip install hatch==1.13.0 click<8.3 + run: pip install "hatch==1.13.0" "click<8.3" - name: Login to Azure for azure-cli authentication uses: azure/login@v2 diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index 14d7e8f61..49a82b7bf 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -37,7 +37,7 @@ jobs: - name: Run unit tests run: | - pip install hatch==1.13.0 click<8.3 + pip install "hatch==1.13.0" "click<8.3" make test fmt: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index b3b690cf8..b2f1e4b17 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -27,7 +27,7 @@ jobs: - name: Build wheels run: | - pip install hatch==1.13.0 click<8.3 + pip install "hatch==1.13.0" "click<8.3" hatch build - name: Github release From 99e6e2bfc41ed9bf16d811aa991b0d836e4d9841 Mon Sep 17 00:00:00 2001 From: Marcin Wojtyczka Date: Fri, 19 Sep 2025 14:20:13 +0200 Subject: [PATCH 29/31] updated tool demo --- demos/dqx_demo_tool.py | 47 +++++++++++++++++++++++++++++++++++------- 1 file changed, 39 insertions(+), 8 deletions(-) diff --git a/demos/dqx_demo_tool.py b/demos/dqx_demo_tool.py index c730d8056..fe4538b5b 100644 --- a/demos/dqx_demo_tool.py +++ b/demos/dqx_demo_tool.py @@ -44,6 +44,14 @@ # MAGIC summary_stats_file: profile_summary_stats.yml # MAGIC warehouse_id: your-warehouse-id # MAGIC ``` +# MAGIC +# MAGIC If you install DQX using custom installation path you must update `custom_install_path` variable below. Installation using custom path is required when using [group assigned cluster](https://docs.databricks.com/aws/en/compute/group-access)! + +# COMMAND ---------- + +# Updated the installation path if you install DQX in a custom folder! +custom_install_path: str = "" +dbutils.widgets.text("dqx_custom_installation_path", custom_install_path, "DQX Custom Installation Path") # COMMAND ---------- @@ -107,8 +115,14 @@ import glob import os -user_name = spark.sql("select current_user() as user").collect()[0]["user"] -default_dqx_installation_path = f"/Workspace/Users/{user_name}/.dqx" +if custom_install_path: + default_dqx_installation_path = custom_install_path + print(f"Using custom installation path: {custom_install_path}") +else: + user_name = spark.sql("select current_user() as user").collect()[0]["user"] + default_dqx_installation_path = f"/Workspace/Users/{user_name}/.dqx" + print(f"Using default user's home installation path: {default_dqx_installation_path}") + default_dqx_product_name = "dqx" dbutils.widgets.text("dqx_installation_path", default_dqx_installation_path, "DQX Installation Folder") @@ -116,6 +130,7 @@ dqx_wheel_files_path = f"{dbutils.widgets.get('dqx_installation_path')}/wheels/databricks_labs_dqx-*.whl" dqx_wheel_files = glob.glob(dqx_wheel_files_path) + try: dqx_latest_wheel = max(dqx_wheel_files, key=os.path.getctime) except: @@ -126,6 +141,10 @@ # COMMAND ---------- +custom_install_path = dbutils.widgets.get('dqx_custom_installation_path') or None + +# COMMAND ---------- + # MAGIC %md # MAGIC ### Run profiler workflow to generate quality rule candidates # MAGIC @@ -162,7 +181,9 @@ dq_engine = DQEngine(ws) # load the run configuration -run_config = RunConfigLoader(ws).load_run_config(run_config_name="default", product_name=dqx_product_name) +run_config = RunConfigLoader(ws).load_run_config( + run_config_name="default", product_name=dqx_product_name, install_folder=custom_install_path +) # read the input data, limit to 1000 rows for demo purpose input_df = read_input_data(spark, run_config.input_config).limit(1000) @@ -180,7 +201,10 @@ print(yaml.safe_dump(checks)) # save generated checks to location specified in the default run configuration inside workspace installation folder -dq_engine.save_checks(checks, config=InstallationChecksStorageConfig(run_config_name="default", product_name=dqx_product_name)) +dq_engine.save_checks(checks, config=InstallationChecksStorageConfig( + run_config_name="default", product_name=dqx_product_name, install_folder=custom_install_path + ) +) # or save checks in arbitrary workspace location #dq_engine.save_checks(checks, config=WorkspaceFileChecksStorageConfig(location="/Shared/App1/checks.yml")) @@ -245,7 +269,10 @@ dq_engine = DQEngine(WorkspaceClient()) # save checks to location specified in the default run configuration inside workspace installation folder -dq_engine.save_checks(checks, config=InstallationChecksStorageConfig(run_config_name="default", product_name=dqx_product_name)) +dq_engine.save_checks(checks, config=InstallationChecksStorageConfig( + run_config_name="default", product_name=dqx_product_name, install_folder=custom_install_path + ) +) # or save checks in arbitrary workspace location #dq_engine.save_checks(checks, config=WorkspaceFileChecksStorageConfig(location="/Shared/App1/checks.yml")) @@ -267,7 +294,9 @@ dq_engine = DQEngine(WorkspaceClient()) # load the run configuration -run_config = RunConfigLoader(ws).load_run_config(run_config_name="default", assume_user=True, product_name=dqx_product_name) +run_config = RunConfigLoader(ws).load_run_config( + run_config_name="default", assume_user=True, product_name=dqx_product_name, install_folder=custom_install_path +) # read the data, limit to 1000 rows for demo purpose bronze_df = read_input_data(spark, run_config.input_config).limit(1000) @@ -276,8 +305,10 @@ bronze_transformed_df = bronze_df.filter("vendor_id in (1, 2)") # load checks from location defined in the run configuration - -checks = dq_engine.load_checks(config=InstallationChecksStorageConfig(assume_user=True, run_config_name="default", product_name=dqx_product_name)) +checks = dq_engine.load_checks(config=InstallationChecksStorageConfig( + assume_user=True, run_config_name="default", product_name=dqx_product_name, install_folder=custom_install_path + ) +) # or load checks from arbitrary workspace file #checks = dq_engine.load_checks(config=WorkspaceFileChecksStorageConfig(location="/Shared/App1/checks.yml")) From b5d332672f71d9bbca2c1427c5fe830d082da6df Mon Sep 17 00:00:00 2001 From: Marcin Wojtyczka Date: Fri, 19 Sep 2025 14:22:59 +0200 Subject: [PATCH 30/31] added comments --- .github/workflows/acceptance.yml | 4 ++++ .github/workflows/docs-release.yml | 1 + .github/workflows/downstreams.yml | 1 + .github/workflows/nightly.yml | 5 +++++ .github/workflows/performance.yml | 1 + .github/workflows/push.yml | 1 + .github/workflows/release.yml | 1 + 7 files changed, 14 insertions(+) diff --git a/.github/workflows/acceptance.yml b/.github/workflows/acceptance.yml index 40233904f..58310ee29 100644 --- a/.github/workflows/acceptance.yml +++ b/.github/workflows/acceptance.yml @@ -41,6 +41,7 @@ jobs: python-version: '3.12' - name: Install hatch + # click 8.3+ introduced bug for hatch run: pip install "hatch==1.13.0" "click<8.3" - name: Run unit tests and generate test coverage report @@ -93,6 +94,7 @@ jobs: python-version: '3.12' - name: Install hatch + # click 8.3+ introduced bug for hatch run: pip install "hatch==1.13.0" "click<8.3" - name: Run integration tests on serverless cluster @@ -125,6 +127,7 @@ jobs: python-version: '3.12' - name: Install hatch + # click 8.3+ introduced bug for hatch run: pip install "hatch==1.13.0" "click<8.3" - name: Install Databricks CLI @@ -177,6 +180,7 @@ jobs: python-version: '3.12' - name: Install hatch + # click 8.3+ introduced bug for hatch run: pip install "hatch==1.13.0" "click<8.3" - name: Install Databricks CLI diff --git a/.github/workflows/docs-release.yml b/.github/workflows/docs-release.yml index 07a6ce1fa..a08228f51 100644 --- a/.github/workflows/docs-release.yml +++ b/.github/workflows/docs-release.yml @@ -28,6 +28,7 @@ jobs: - name: Install Hatch run: | + # click 8.3+ introduced bug for hatch pip install "hatch==1.13.0" "click<8.3" - uses: actions/setup-node@v4 diff --git a/.github/workflows/downstreams.yml b/.github/workflows/downstreams.yml index c2551f6c2..4f497c3d8 100644 --- a/.github/workflows/downstreams.yml +++ b/.github/workflows/downstreams.yml @@ -43,6 +43,7 @@ jobs: - name: Install toolchain run: | + # click 8.3+ introduced bug for hatch pip install "hatch==1.13.0" "click<8.3" - name: Check downstream compatibility diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 2e9ea560a..aa73286b6 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -32,6 +32,7 @@ jobs: python-version: '3.12' - name: Install hatch + # click 8.3+ introduced bug for hatch run: pip install "hatch==1.13.0" "click<8.3" - name: Run unit tests and generate test coverage report @@ -81,6 +82,7 @@ jobs: python-version: '3.12' - name: Install hatch + # click 8.3+ introduced bug for hatch run: pip install "hatch==1.13.0" "click<8.3" - name: Run integration tests on serverless cluster @@ -113,6 +115,7 @@ jobs: python-version: '3.12' - name: Install hatch + # click 8.3+ introduced bug for hatch run: pip install "hatch==1.13.0" "click<8.3" - name: Install Databricks CLI @@ -164,6 +167,7 @@ jobs: python-version: '3.12' - name: Install hatch + # click 8.3+ introduced bug for hatch run: pip install "hatch==1.13.0" "click<8.3" - name: Install Databricks CLI @@ -217,6 +221,7 @@ jobs: python-version: '3.12' - name: Install hatch + # click 8.3+ introduced bug for hatch run: pip install "hatch==1.13.0" "click<8.3" - name: Login to Azure for azure-cli authentication diff --git a/.github/workflows/performance.yml b/.github/workflows/performance.yml index 7c1c9d650..cf84000c7 100644 --- a/.github/workflows/performance.yml +++ b/.github/workflows/performance.yml @@ -43,6 +43,7 @@ jobs: cache-dependency-path: '**/pyproject.toml' - name: Install hatch + # click 8.3+ introduced bug for hatch run: pip install "hatch==1.13.0" "click<8.3" - name: Login to Azure for azure-cli authentication diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index 49a82b7bf..d45a722ed 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -37,6 +37,7 @@ jobs: - name: Run unit tests run: | + # click 8.3+ introduced bug for hatch pip install "hatch==1.13.0" "click<8.3" make test diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index b2f1e4b17..469b80b36 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -27,6 +27,7 @@ jobs: - name: Build wheels run: | + # click 8.3+ introduced bug for hatch pip install "hatch==1.13.0" "click<8.3" hatch build From 22854c39884989b4558983a13435da6bbf0128e1 Mon Sep 17 00:00:00 2001 From: Marcin Wojtyczka Date: Fri, 19 Sep 2025 14:32:38 +0200 Subject: [PATCH 31/31] updated docs --- docs/dqx/docs/dev/contributing.mdx | 9 +++++++++ tests/conftest.py | 2 +- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/docs/dqx/docs/dev/contributing.mdx b/docs/dqx/docs/dev/contributing.mdx index afd108a43..4c058b416 100644 --- a/docs/dqx/docs/dev/contributing.mdx +++ b/docs/dqx/docs/dev/contributing.mdx @@ -333,6 +333,15 @@ git push --force-with-lease origin HEAD If you encounter any package dependency errors after `git pull`, run `make clean` +### Resolving Hatch JSON TypeError + +If you encounter an error like: +```text +TypeError: the JSON object must be str, bytes or bytearray, not Sentinel +``` + +you can resolve it by downgrading the Click package to a compatible version that works with hatch: `pip install "click<8.3"` + ### Common fixes for `mypy` errors See https://mypy.readthedocs.io/en/stable/cheat_sheet_py3.html for more details diff --git a/tests/conftest.py b/tests/conftest.py index 37ca657b2..5766c0846 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -22,7 +22,7 @@ @pytest.fixture def debug_env_name(): - return "ws2" # Specify the name of the debug environment from ~/.databricks/debug-env.json + return "ws" # Specify the name of the debug environment from ~/.databricks/debug-env.json @pytest.fixture