Adding Lakebase checks storage backend

docs/dqx/docs/guide/data_profiling.mdx

@@ -262,7 +262,7 @@ When running the profiler workflow using Databricks API or UI, you have the same
   - If the `checks_location` in the run config points to a table, the checks will be saved to that table.
     If the `checks_location` in the run config points to a file, file name is replaced with "<input_table>.yml". In addition, if the location is specified as a relative path, it is prefixed with the workspace installation folder.
     For example:
-    - If "checks_location=catalog.schema.table", the location will be resolved to "catalog.schema.table".
+    - If "checks_location=catalog.schema.table", the location will be resolved to "catalog.schema.table" or "database.schema.table" in case of using Lakebase to store checks.
     - If "checks_location=folder/checks.yml", the  location will be resolved to "install_folder/folder/<input_table>.yml".
     - If "checks_location=/App/checks.yml", the location will be resolved to "/App/<input_table>.yml".
     - If "checks_location=/Volume/catalog/schema/folder/checks.yml", the location will be resolved to "/Volume/catalog/schema/folder/<input_table>.yml".

docs/dqx/docs/guide/quality_checks_apply.mdx

@@ -573,7 +573,7 @@ When running the quality checker workflow using Databricks API or UI, you have t
   - If the `checks_location` in the run config points to a table, the checks will be directly loaded from that table.
     If the `checks_location` in the run config points to a file, file name is replaced with "<input_table>.yml". In addition, if the location is specified as a relative path, it is prefixed with the workspace installation folder.
     For example:
-    - If "checks_location=catalog.schema.table", the location will be resolved to "catalog.schema.table".
+    - If "checks_location=catalog.schema.table", the location will be resolved to "catalog.schema.table" or "database.schema.table" in case of using Lakebase to store checks.
     - If "checks_location=folder/checks.yml", the  location will be resolved to "install_folder/folder/<input_table>.yml".
     - If "checks_location=/App/checks.yml", the location will be resolved to "/App/<input_table>.yml".
     - If "checks_location=/Volume/catalog/schema/folder/checks.yml", the location will be resolved to "/Volume/catalog/schema/folder/<input_table>.yml".
@@ -688,7 +688,7 @@ When running the e2e workflow using Databricks API or UI, you have the same exec
   - If the `checks_location` in the run config points to a table, the checks will be directly loaded from that table.
     If the `checks_location` in the run config points to a file, file name is replaced with <input_table>.yml. In addition, if the location is specified as a relative path, it is prefixed with the workspace installation folder.
     For example:
-    - If "checks_location=catalog.schema.table", the location will be resolved to "catalog.schema.table".
+    - If "checks_location=catalog.schema.table", the location will be resolved to "catalog.schema.table" or "database.schema.table" in case of using Lakebase to store checks.
     - If "checks_location=folder/checks.yml", the  location will be resolved to "install_folder/folder/<input_table>.yml".
     - If "checks_location=/App/checks.yml", the location will be resolved to "/App/<input_table>.yml".
     - If "checks_location=/Volume/catalog/schema/folder/checks.yml", the location will be resolved to "/Volume/catalog/schema/folder/<input_table>.yml".

docs/dqx/docs/guide/quality_checks_storage.mdx

@@ -22,6 +22,13 @@ Saving and loading methods accept a storage backend configuration as input. The
     * `mode`: (optional) write mode for saving checks (`overwrite` or `append`, default is `overwrite`). The `overwrite` mode will only replace checks for the specific run config and not all checks in the table.
 * `VolumeFileChecksStorageConfig`: Unity Catalog Volume (JSON/YAML file). Containing fields:
     * `location`: Unity Catalog Volume file path (JSON or YAML).
+* `LakebaseChecksStorageConfig`: Lakebase table. Containing fields:
+    * `instance_name`: name of the Lakebase instance, e.g., "my-instance".
+    * `user`: user to connect to the Lakebase instance, e.g., "user@domain.com" or Databricks service principal client ID.
+    * `location`: fully-qualified table name in the format "database.schema.table".
+    * `port`: (optional) port on which to connect to the Lakebase instance (use 5432 if not provided).
+    * `run_config_name`: (optional) run configuration name to load (use "default" if not provided).
+    * `mode`: (optional) write mode for saving checks (`overwrite` or `append`, default is `overwrite`). The `overwrite` mode will only replace checks for the specific run config and not all checks in the table.
 * `InstallationChecksStorageConfig`: installation-managed location from the run config, ignores location and infers it from `checks_location` in the run config. Containing fields:
     * `location` (optional): automatically set based on the `checks_location` field from the run configuration.
     * `install_folder`: (optional) installation folder where DQX is installed, only required when custom installation folder is used.
@@ -49,7 +56,8 @@ If you create checks as a list of `DQRule` objects, you can convert them to meta
       WorkspaceFileChecksStorageConfig,
       InstallationChecksStorageConfig,
       TableChecksStorageConfig,
-      VolumeFileChecksStorageConfig
+      VolumeFileChecksStorageConfig,
+      LakebaseChecksStorageConfig,
     )
     from databricks.sdk import WorkspaceClient
 
@@ -81,6 +89,9 @@ If you create checks as a list of `DQRule` objects, you can convert them to meta
     # save checks as a YAML in a Unity Catalog Volume location (overwrite the file)
     dq_engine.save_checks(checks, config=VolumeFileChecksStorageConfig(location="/Volumes/dq/config/checks_volume/App1/checks.yml"))
 
+    # save checks as a Lakebase table using a Databricks service principal
+    dq_engine.save_checks(checks, config=LakebaseChecksStorageConfig(instance_name="my-instance", user="00000000-0000-0000-0000-000000000000", location="dqx.config.checks"))
+
     # save checks as a YAML file or table defined in 'checks_location' of the run config
     # only works if DQX is installed in the workspace
     dq_engine.save_checks(checks, config=InstallationChecksStorageConfig(assume_user=True, run_config_name="default"))
@@ -194,7 +205,8 @@ If you create checks as a list of DQRule objects, you can convert them using the
       WorkspaceFileChecksStorageConfig,
       InstallationChecksStorageConfig,
       TableChecksStorageConfig,
-      VolumeFileChecksStorageConfig
+      VolumeFileChecksStorageConfig,
+      LakebaseChecksStorageConfig,
     )
     from databricks.sdk import WorkspaceClient
 
@@ -216,6 +228,9 @@ If you create checks as a list of DQRule objects, you can convert them using the
     # load checks from a Unity Catalog Volume
     checks: list[dict] = dq_engine.load_checks(config=VolumeFileChecksStorageConfig(location="/Volumes/dq/config/checks_volume/App1/checks.yml"))
 
+    # load checks from a Lakebase table using a Databricks service principal
+    checks: list[dict] = dq_engine.load_checks(config=LakebaseChecksStorageConfig(instance_name="my-instance", user="00000000-0000-0000-0000-000000000000", location="dqx.config.checks"))
+
     # load checks from a file or table defined in the run config ('checks_location' field)
     # only works if DQX is installed in the workspace
     checks: list[dict] = dq_engine.load_checks(config=InstallationChecksStorageConfig(run_config_name="default"))

pyproject.toml

@@ -29,8 +29,9 @@ classifiers = [
     "Topic :: Utilities",
 ]
 dependencies = ["databricks-labs-blueprint>=0.9.1,<0.10",
-                "databricks-sdk~=0.57",
+                "databricks-sdk~=0.67",
                 "databricks-labs-lsql>=0.5,<=0.16",
+                "sqlalchemy>=1.4,<3.0",
 ]
 
 [project.optional-dependencies]
@@ -101,6 +102,7 @@ dependencies = [
     "dbldatagen~=0.4.0",
     "pyparsing~=3.2.3",
     "jmespath~=1.0.1",
+    "psycopg2-binary~=2.9.10",
 ]
 
 python="3.12" # must match the version required by databricks-connect and python version on the test clusters
@@ -111,7 +113,7 @@ path = ".venv"
 [tool.hatch.envs.default.scripts]
 test        = "pytest tests/unit/ -n 10 --cov --cov-report=xml:coverage-unit.xml --timeout 30 --durations 20"
 coverage    = "pytest tests/ -n 10 --cov --cov-report=html --timeout 600 --durations 20"
-integration = "pytest tests/integration/ -n 10 --cov --cov-report=xml --timeout 1200 --durations 20"
+integration = "pytest tests/integration/ -n 5 --cov --cov-report=xml --timeout 1200 --durations 20"
 e2e         = "pytest tests/e2e/ -n 10 --cov --cov-report=xml --timeout 600 --durations 20"
 perf         = "pytest tests/perf/ -n 10 --cov --cov-report=xml --timeout 600 --durations 20"
 fmt         = ["black .",